R-836: the kernel lane — one-shot boot through the ESP flag, boot good / one self-revert, night step on a told night
gates / gates (push) Successful in 44s

Wrapper layer kernel (stage / reboot / boot / good / revert / cancel / status;
R20-R23), the two GRUB generators in the bundle (option C on the one-shot
entry), the agent's night step and after-boot judge (host health rule + hub
reached, 20 min measured), the signed os_kernel_step (stage only).
Red-proofs: felhom.eu audits/kernel-lane-2026-10-07/A/redproof.txt.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-07 15:18:24 +02:00
parent b17d1c597d
commit d03ab7f1f5
16 changed files with 1955 additions and 16 deletions
+11
View File
@@ -636,6 +636,17 @@ type WireOSUpdate struct {
// HostRelease is the newest approved HOST release (hub v0.131.0, `11` §8 step 3) — a separate set: a version
// approved for the guest is not approved for the host by that fact alone.
HostRelease *WireOSRelease `json:"host_release,omitempty"`
// Kernel is the kernel lane's instruction for THIS box (R-836, `09` §3 decision 172, `11` §5.11): the kernel the
// household was told about, and whether tonight is a told night. Nil (an older hub, or no kernel due) = no kernel
// step. The hub sets Tonight only after the household's mail the day before went out — no mail, no step.
Kernel *WireKernelStep `json:"kernel,omitempty"`
}
// WireKernelStep is the hub's kernel-lane instruction (hub osupdates.KernelBlock — field-exact, cross-repo).
type WireKernelStep struct {
Kver string `json:"kver"` // e.g. "7.0.14-22-pve" — the wrapper refuses any other (R23)
Tonight bool `json:"tonight"` // the household was mailed the day before: tonight's leg may reboot
NotifiedAt string `json:"notified_at,omitempty"` // when that mail went out (RFC 3339), for the log
}
// WireOSRelease is an approved version set; Snapshot is the approval time (YYYYMMDDTHHMMSSZ) the wrapper uses
+409
View File
@@ -0,0 +1,409 @@
package osupdate
// The kernel lane (R-836, `09` §3 decisions 164 + 172, `11` §5.11). The root half is felhom-os-apply's layer "kernel"
// (configs/, its own tests); this file decides WHEN and judges the boot:
//
// - the night leg (Run → runKernel): after a healthy host step, on a night the hub marks as told (the household was
// mailed the day before — no mail, no step): ring 0 STAGES the pending kernel (select pending-kernel, the root-owned
// ring-0 mark) and reboots; ring 1 reboots only a step a signed os_kernel_step staged earlier (KernelStepExecutor).
// - after a boot (KernelAfterBoot, at daemon start): the wrapper says what became of the step. On the new kernel the
// agent JUDGES the boot — the host health rule (`11` §8.2: the Proxmox daemons, the guest running and healthy, the
// tunnel) AND the box reaching the hub — for KernelJudgeWait. Healthy → kernel-good (the new kernel becomes the
// default). Not healthy by the deadline → ONE self-revert (kernel-revert: a reboot into the old kernel, still the
// default). A crash on the new kernel needs nothing from the agent: GRUB already boots the old default.
//
// The host is rebooted by this file only through the wrapper (kernel-reboot, kernel-revert), and only for a staged step.
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"log/slog"
"regexp"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)
// OpKernelStep is the signed op class that STAGES a kernel set on a ring-1 box (it never reboots: the night leg does,
// once the household was told). CC may sign it until the first paying customer (R-530 ruling).
const OpKernelStep = "os_kernel_step"
// DefaultKernelJudgeWait is how long a one-shot boot may take to come back healthy before the agent reverts it ONCE.
// Measured 2026-10-07 (`audits/kernel-lane-2026-10-07/B/`): every container healthy 68 s after a reboot on demo-felhom,
// 272 s on demo-hp; 20 minutes stays under the hub's 30-minute host_stale (a box that never comes back alarms after it).
const DefaultKernelJudgeWait = 20 * time.Minute
var kverRE = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve$`)
// KernelView is the wrapper's kernel object (felhom-os-apply Kernel.view).
type KernelView struct {
Running string `json:"running"`
Default string `json:"default"`
Flag *string `json:"flag"`
Phase string `json:"phase"`
From string `json:"from"`
To string `json:"to"`
SelfRevertUsed bool `json:"self_revert_used"`
Reason string `json:"reason"`
VMID int `json:"vmid"` // the customer guest the step was staged for (the health rule's guest)
}
func parseKernel(raw json.RawMessage) KernelView {
var v KernelView
_ = json.Unmarshal(raw, &v)
return v
}
// kernelPlan is one kernel-layer wrapper call.
func kernelPlan(mode string, vmid int, extra map[string]any) map[string]any {
p := map[string]any{"release_id": "kernel", "layer": LayerKernel, "lane": "slow", "vmid": vmid, "mode": mode,
"packages": []Package{}}
for k, v := range extra {
p[k] = v
}
return p
}
// KernelStatus reads the kernel lane's state (read only).
func (l *Leg) KernelStatus(ctx context.Context, vmid int) (KernelView, error) {
wr, err := l.call(ctx, "kstatus"+l.now().UTC().Format("150405"), kernelPlan("kernel-status", vmid, nil))
if err != nil {
return KernelView{}, err
}
if wr.refused() {
return KernelView{}, fmt.Errorf("kernel-status refused: %s", wr.Refused)
}
return parseKernel(wr.Kernel), nil
}
// kernelDue reports whether tonight's leg may take a kernel step, and why not.
func kernelDue(blk hub.WireOSUpdate, trigger string) (bool, string) {
switch {
case trigger != "night":
return false, "a kernel step runs only in the night leg (never a debug pass)"
case blk.Kernel == nil:
return false, "the hub names no kernel step for this box"
case !blk.Enabled:
return false, "OS updates are switched off for this box"
case !kverRE.MatchString(blk.Kernel.Kver):
return false, "the hub's kernel " + blk.Kernel.Kver + " is not a kernel version"
case !blk.Kernel.Tonight:
return false, "the household has not been told about tonight (no mail, no step — `09` §3 decision 172)"
}
return true, ""
}
// runKernel is the night leg's last step. It returns the stage report (Layer "" when nothing ran). On success the box
// is rebooting when it returns.
func (l *Leg) runKernel(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", trigger, "ring", blk.Ring)
if ok, why := kernelDue(blk, trigger); !ok {
lg.Info("osupdate: kernel step skipped — " + why)
return Report{}
}
want := blk.Kernel.Kver
st, err := l.KernelStatus(ctx, vmid)
if err != nil {
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
Mode: "apply", Outcome: "failed", HealthReason: "kernel status unreadable: " + err.Error()})
}
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply", ReleaseID: want}
switch {
case st.Phase == "staged" && st.To == want:
lg.Info("osupdate: kernel step — a staged kernel waits for tonight", "from", st.From, "to", st.To)
rep.Outcome, rep.Healthy = "staged", true
case blk.Ring != 0:
lg.Info("osupdate: kernel step skipped — ring 1 boots only a kernel a signed os_kernel_step staged", "phase", st.Phase, "staged", st.To, "want", want)
return Report{}
default:
wr, cerr := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": "ring0-" + runID,
"select": "pending-kernel", "expect_kver": want, "run_id": runID, "trigger": trigger, "ring": blk.Ring}))
rep.unsent = reportFile(l.planDir(), runID, LayerKernel, "apply")
rep.Kernel = rawOrNil(wr.Kernel)
switch {
case cerr != nil:
rep.Outcome, rep.HealthReason = "failed", cerr.Error()
return l.finish(ctx, lg, rep)
case wr.refused():
rep.Outcome, rep.Refused = "refused", wr.Refused
return l.finish(ctx, lg, rep)
case wr.failed():
rep.Outcome, rep.Refused = "failed", wr.Failed
return l.finish(ctx, lg, rep)
case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0:
rep.Outcome, rep.Healthy = "nothing", true
return l.finish(ctx, lg, rep)
}
rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds
rep.RebootNeeded = true
}
rep = l.finish(ctx, lg, rep) // the hub hears "staged" BEFORE the box goes down
wr, err := l.call(ctx, runID, kernelPlan("kernel-reboot", vmid, nil))
switch {
case err != nil:
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
Mode: "kernel-reboot", ReleaseID: want, Outcome: "failed", HealthReason: "kernel-reboot: " + err.Error()})
case wr.refused() || wr.failed():
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
Mode: "kernel-reboot", ReleaseID: want, Outcome: "refused", Refused: firstRaw(wr.Refused, wr.Failed),
Kernel: rawOrNil(wr.Kernel)})
}
lg.Warn("osupdate: kernel step — the box restarts now for its one-shot boot", "to", want)
return rep
}
func firstRaw(a, b json.RawMessage) json.RawMessage {
if r := rawOrNil(a); r != nil {
return r
}
return rawOrNil(b)
}
// KernelJudge is what KernelAfterBoot needs besides the leg: the hub reachability probe is the "judging" report itself.
type KernelJudge struct {
Wait time.Duration // default DefaultKernelJudgeWait
Poll time.Duration // default 30 s
}
// KernelAfterBoot runs once at daemon start: what became of a kernel step across the boot. On the new kernel it judges
// the boot (blocking up to the wait — run it in a goroutine). vmid 0 = the guest the step recorded (it may not run yet).
func (l *Leg) KernelAfterBoot(ctx context.Context, vmid int, j KernelJudge) Report {
runID := "boot-" + l.now().UTC().Format("20060102T150405Z")
lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid)
wr, err := l.call(ctx, runID, kernelPlan("kernel-boot", vmid, nil))
if err != nil {
lg.Warn("osupdate: kernel after-boot check failed", "err", err)
return Report{}
}
if wr.refused() {
lg.Info("osupdate: kernel after-boot check refused (an older wrapper, or a BYO host)", "refused", string(wr.Refused))
return Report{}
}
v := parseKernel(wr.Kernel)
if vmid <= 0 {
vmid = v.VMID // after a boot the guest may not run yet — the step's own record names it
}
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid, Mode: "kernel-boot",
ReleaseID: v.To, Kernel: rawOrNil(wr.Kernel)}
switch wr.KernelEvent {
case "fell_back":
rep.Outcome, rep.HealthReason = "fell_back", v.Reason
lg.Warn("osupdate: kernel step FELL BACK — the new kernel did not come up; the box runs the old one", "from", v.From, "to", v.To)
return l.finish(ctx, lg, rep)
case "self_reverted":
rep.Outcome, rep.HealthReason = "self_reverted", v.Reason
lg.Warn("osupdate: kernel step SELF-REVERTED — back on the old kernel", "from", v.From, "to", v.To, "reason", v.Reason)
return l.finish(ctx, lg, rep)
case "revert_failed":
rep.Outcome, rep.HealthReason = "revert_failed", v.Reason
lg.Error("osupdate: kernel self-revert came back on the NEW kernel — no second revert; the operator decides", "to", v.To)
return l.finish(ctx, lg, rep)
case "judging":
return l.judgeKernel(ctx, runID, vmid, v, wr.HealthBefore, j, lg)
}
return Report{}
}
// KernelVerdict is THE one-shot boot rule (R-836; pinned by TestKernelVerdict): the host health rule (`11` §8.2 —
// the Proxmox daemons and the agent active, the customer guest running and its own rule passing, the tunnel running)
// AND the box reached the hub since this boot.
func KernelVerdict(before, after *Health, tunnel string, hubReached bool) (bool, string) {
if ok, why := HostHealthVerdict(before, after, tunnel); !ok {
return false, why
}
if !hubReached {
return false, "the box has not reached the hub since the boot"
}
return true, ""
}
func (l *Leg) judgeKernel(ctx context.Context, runID string, vmid int, v KernelView, before *Health, j KernelJudge, lg *slog.Logger) Report {
wait, poll := j.Wait, j.Poll
if wait <= 0 {
wait = DefaultKernelJudgeWait
}
if poll <= 0 {
poll = 30 * time.Second
}
lg.Info("osupdate: kernel step — judging the one-shot boot", "from", v.From, "to", v.To, "wait", wait.String())
start := l.now()
deadline := start.Add(wait)
hubReached := false
var why string
for {
if !hubReached && l.Hub != nil {
// the hub's reachability IS this report reaching it (and the operator sees the box is back on the new kernel)
body, _ := json.Marshal(Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
Mode: "kernel-boot", ReleaseID: v.To, Outcome: "judging", Kernel: mustRaw(v)})
rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
if err := l.Hub.PostOSReport(rctx, body); err == nil {
hubReached = true
lg.Info("osupdate: kernel step — the box reached the hub on the new kernel", "after", l.now().Sub(start).Round(time.Second).String())
}
cancel()
}
var h *Health
hr, err := l.call(ctx, runID, kernelPlan("health", vmid, nil))
switch {
case err != nil:
why = "no health reading: " + err.Error()
case hr.refused():
why = "no health reading: " + string(hr.Refused)
default:
h = hr.Health
}
if h != nil {
t := hub.TunnelUnknown
if l.Tunnel != nil {
t, _ = l.Tunnel.Status(ctx)
}
var ok bool
ok, why = KernelVerdict(before, h, t, hubReached)
if ok {
return l.kernelGood(ctx, runID, vmid, v, start, lg)
}
}
if !l.now().Before(deadline) || ctx.Err() != nil {
break
}
l.sleep(ctx, poll)
}
if ctx.Err() != nil {
lg.Warn("osupdate: kernel judging stopped (the agent is stopping) — the next start judges again", "reason", why)
return Report{}
}
// not healthy by the deadline: tell the hub (best effort), then ONE self-revert into the old kernel
rep := l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
Mode: "kernel-revert", ReleaseID: v.To, Outcome: "health_failed", HealthReason: why + " — reverting to " + v.From,
Kernel: mustRaw(v)})
lg.Error("osupdate: kernel step — the one-shot boot is NOT healthy; restarting ONCE into the old kernel", "reason", why,
"waited", wait.String(), "from", v.From, "to", v.To)
wr, err := l.call(ctx, runID, kernelPlan("kernel-revert", vmid, map[string]any{"reason": truncate(why, 280)}))
if err != nil || wr.refused() || wr.failed() {
lg.Error("osupdate: kernel self-revert did not start — the box stays on the new kernel; the operator decides",
"err", err, "refused", string(firstRaw(wr.Refused, wr.Failed)))
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
Mode: "kernel-revert", ReleaseID: v.To, Outcome: "revert_failed", Refused: firstRaw(wr.Refused, wr.Failed),
HealthReason: "the self-revert did not start"})
}
return rep
}
func (l *Leg) kernelGood(ctx context.Context, runID string, vmid int, v KernelView, start time.Time, lg *slog.Logger) Report {
wr, err := l.call(ctx, runID, kernelPlan("kernel-good", vmid, nil))
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid, Mode: "kernel-good",
ReleaseID: v.To}
switch {
case err != nil:
rep.Outcome, rep.HealthReason = "failed", "kernel-good: "+err.Error()
case wr.refused() || wr.failed():
rep.Outcome, rep.Refused, rep.HealthReason = "failed", firstRaw(wr.Refused, wr.Failed), "kernel-good did not move the default"
default:
rep.Outcome, rep.Healthy = "applied", true
rep.HealthReason = fmt.Sprintf("healthy %s after the agent started; the new kernel is the default", l.now().Sub(start).Round(time.Second))
}
rep.Kernel = rawOrNil(wr.Kernel)
lg.Info("osupdate: kernel step — "+rep.Outcome, "to", v.To, "reason", rep.HealthReason)
return l.finish(ctx, lg, rep)
}
func mustRaw(v any) json.RawMessage {
b, _ := json.Marshal(v)
return b
}
func truncate(s string, n int) string {
if len(s) <= n {
return s
}
return s[:n]
}
// KernelStepParams are a signed os_kernel_step's params: the exact kernel set (the wrapper compares it with the plan).
type KernelStepParams struct {
ReleaseID string `json:"release_id"`
Packages []Package `json:"packages"`
Kver string `json:"kver"`
VMID int `json:"vmid,omitempty"`
}
// KernelStepExecutor STAGES a verified os_kernel_step (signedjobs.Executor) under the heavy-op gate. It never reboots:
// the night leg reboots a staged kernel on a night the household was told about.
type KernelStepExecutor struct {
Leg *Leg
Guest func(ctx context.Context) (int, error)
Gate func(ctx context.Context) (release func(), err error)
}
// Execute implements signedjobs.Executor.
func (e KernelStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
if op != OpKernelStep {
return signedjobs.ErrNoExecutor
}
so, ok := signedjobs.SignedOpFrom(ctx)
if !ok {
return fmt.Errorf("os_kernel_step: no signed envelope in the context — the wrapper could not verify it")
}
var p KernelStepParams
if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 || !kverRE.MatchString(p.Kver) {
return fmt.Errorf("os_kernel_step: params must name the kernel set and its kver: %v", err)
}
vmid := p.VMID
if vmid == 0 {
if e.Guest == nil {
return fmt.Errorf("os_kernel_step: no vmid and no guest finder")
}
v, err := e.Guest(ctx)
if err != nil {
return fmt.Errorf("os_kernel_step: find the customer guest: %w", err)
}
vmid = v
}
if e.Gate != nil {
release, err := e.Gate(ctx)
if err != nil {
return fmt.Errorf("os_kernel_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
}
defer release()
}
rep := e.Leg.StageKernelSigned(ctx, vmid, p, so.Blob, string(so.Sig))
if rep.Outcome == "staged" || rep.Outcome == "nothing" {
return nil
}
return fmt.Errorf("os_kernel_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
}
// StageKernelSigned stages a signed kernel set (ring 1): install + flag, no reboot.
func (l *Leg) StageKernelSigned(ctx context.Context, vmid int, p KernelStepParams, blob []byte, sig string) Report {
unlock := l.lockPass(true)
defer unlock()
l.sendUnsentLocked(ctx) // R-868
runID := l.now().UTC().Format("20060102T150405Z")
rid := p.ReleaseID
if rid == "" {
rid = "signed-" + runID
}
lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", "signed", "release", rid)
wr, err := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": rid, "select": "listed",
"packages": p.Packages, "expect_kver": p.Kver, "run_id": runID, "trigger": "signed", "ring": l.Block().Ring,
"signed": map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}}))
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "signed", Ring: l.Block().Ring, VMID: vmid, Mode: "apply",
ReleaseID: rid, Kernel: rawOrNil(wr.Kernel), unsent: reportFile(l.planDir(), runID, LayerKernel, "apply")}
switch {
case err != nil:
rep.Outcome, rep.HealthReason = "failed", err.Error()
case wr.refused():
rep.Outcome, rep.Refused = "refused", wr.Refused
case wr.failed():
rep.Outcome, rep.Refused = "failed", wr.Failed
case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0:
rep.Outcome, rep.Healthy = "nothing", true
default:
rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds
rep.RebootNeeded = true
}
return l.finish(ctx, lg, rep)
}
+314
View File
@@ -0,0 +1,314 @@
package osupdate
import (
"context"
"encoding/base64"
"encoding/json"
"strings"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)
// ---- the kernel lane (R-836, `09` §3 decision 172, `11` §5.11) ----
const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve"
func kview(phase string) json.RawMessage {
return mustRaw(KernelView{Running: kOld, Default: kOld, Phase: phase, From: kOld, To: kNew, VMID: 9201})
}
func tonight(ring int) *hub.WireOSUpdate {
return &hub.WireOSUpdate{Ring: ring, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}
}
func kernelCalls(w *fakeWrapper) []string {
var m []string
for _, p := range w.plans {
if p["layer"] == LayerKernel {
m = append(m, p["mode"].(string))
}
}
return m
}
// Ring 0, a told night: after the healthy host step the leg stages the pending kernel (select pending-kernel, the
// kernel the household was told about), tells the hub "staged", THEN reboots — the kernel step ends the night.
func TestKernel_Ring0ToldNightStagesThenReboots(t *testing.T) {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{
"kernel-status": {{Kernel: kview("none")}},
"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}},
"kernel-reboot": {{Kernel: kview("oneshot")}},
}}
l, h := newLeg(t, w, tonight(0))
p := l.Run(context.Background(), 9201, "night")
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply,kernel-reboot" {
t.Fatalf("kernel calls = %s", got)
}
var ap map[string]any
for _, x := range w.plans {
if x["layer"] == LayerKernel && x["mode"] == "apply" {
ap = x
}
}
if ap["select"] != "pending-kernel" || ap["expect_kver"] != kNew || ap["lane"] != "slow" {
t.Fatalf("stage plan = %v", ap)
}
if p.Kernel.Outcome != "staged" || !p.Kernel.Healthy {
t.Fatalf("kernel report = %+v", p.Kernel)
}
last := h.reports[len(h.reports)-1]
if last.Layer != LayerKernel || last.Outcome != "staged" {
t.Fatalf("the hub must hear 'staged' before the reboot: %+v", h.reports)
}
// the kernel step is the LAST wrapper call of the night
if lp := w.plans[len(w.plans)-1]; lp["layer"] != LayerKernel || lp["mode"] != "kernel-reboot" {
t.Fatalf("the reboot must end the night, last call = %v", lp)
}
}
// No mail, no step: a kernel the household was NOT told about never runs; nor in a debug pass; nor without a block.
// COMPANION RED-PROOF (observed): drop the `!blk.Kernel.Tonight` case in kernelDue → the first sub-case fails.
func TestKernel_NoMailNoStep(t *testing.T) {
cases := map[string]struct {
blk *hub.WireOSUpdate
trigger string
}{
"not told": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: false}}, "night"},
"debug pass": {tonight(0), "debug"},
"no block": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night"},
"switch off": {&hub.WireOSUpdate{Ring: 0, Enabled: false, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}, "night"},
"bad kver": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: "7.0; reboot", Tonight: true}}, "night"},
}
for name, c := range cases {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, c.blk)
p := l.Run(context.Background(), 9201, c.trigger)
if len(kernelCalls(w)) != 0 || p.Kernel.Layer != "" {
t.Fatalf("%s: a kernel step ran: %v", name, kernelCalls(w))
}
}
}
// The kernel step needs a healthy host step on an appliance, and a healthy Proxmox step when one ran.
func TestKernel_SkippedWithoutHealthyEarlierSteps(t *testing.T) {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, tonight(0))
l.Appliance = false
l.Run(context.Background(), 9201, "night")
if len(kernelCalls(w)) != 0 {
t.Fatalf("a BYO box took a kernel step: %v", kernelCalls(w))
}
w2 := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}},
PVEManager: "9.2.2"}}} // pveversion still old → the pve step is unhealthy
l2, _ := newLeg(t, w2, tonight(0))
l2.Run(context.Background(), 9201, "night")
if len(kernelCalls(w2)) != 0 {
t.Fatalf("a kernel step ran after an unhealthy Proxmox step: %v", kernelCalls(w2))
}
}
// Ring 1 reboots only a kernel a signed job staged — never stages one itself in the night leg.
func TestKernel_Ring1RebootsOnlyASignedStage(t *testing.T) {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}}}
l, _ := newLeg(t, w, tonight(1))
l.Run(context.Background(), 9201, "night")
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status" {
t.Fatalf("ring 1 without a staged kernel: calls = %s", got)
}
w2 := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("staged")}},
"kernel-reboot": {{Kernel: kview("oneshot")}}}}
l2, _ := newLeg(t, w2, tonight(1))
p := l2.Run(context.Background(), 9201, "night")
if got := strings.Join(kernelCalls(w2), ","); got != "kernel-status,kernel-reboot" || p.Kernel.Outcome != "staged" {
t.Fatalf("ring 1 with a staged kernel: calls = %s report = %+v", got, p.Kernel)
}
}
// A refused stage never reboots.
func TestKernel_RefusedStageNeverReboots(t *testing.T) {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}},
"apply": {{Refused: json.RawMessage(`{"code":"R20","reason":"/boot/efi is not a mounted vfat ESP"}`)}}}}
l, h := newLeg(t, w, tonight(0))
p := l.Run(context.Background(), 9201, "night")
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply" || p.Kernel.Outcome != "refused" {
t.Fatalf("calls = %s report = %+v", got, p.Kernel)
}
if last := h.reports[len(h.reports)-1]; last.Layer != LayerKernel || last.Outcome != "refused" {
t.Fatalf("the hub must hear the refusal: %+v", last)
}
}
// THE one-shot boot rule: the host rule AND the hub reached. COMPANION RED-PROOF (observed): drop the hubReached
// check in KernelVerdict → the second case fails.
func TestKernelVerdict(t *testing.T) {
if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, true); !ok {
t.Fatalf("a healthy boot read unhealthy: %s", why)
}
if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, false); ok || !strings.Contains(why, "hub") {
t.Fatalf("a box that has not reached the hub must not pass: ok=%v %q", ok, why)
}
down := hostOK()
down.GuestRunning = new(bool)
if ok, _ := KernelVerdict(hostOK(), down, hub.TunnelRunning, true); ok {
t.Fatal("a guest that does not run must fail")
}
if ok, _ := KernelVerdict(hostOK(), hostOK(), hub.TunnelUnknown, true); ok {
t.Fatal("an unknown tunnel must fail (the host rule)")
}
}
func judgingLeg(t *testing.T, w *fakeWrapper) (*Leg, *fakeHub) {
if w.kernelRep == nil {
w.kernelRep = map[string][]WrapperReport{}
}
if _, ok := w.kernelRep["kernel-boot"]; !ok {
w.kernelRep["kernel-boot"] = []WrapperReport{{KernelEvent: "judging", Kernel: mustRaw(KernelView{Running: kNew,
Default: kOld, Phase: "judging", From: kOld, To: kNew, VMID: 9201}), HealthBefore: hostOK()}}
}
return newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
}
// A healthy one-shot boot: the hub hears "judging", then kernel-good, then "applied".
func TestKernelAfterBoot_HealthyBecomesTheDefault(t *testing.T) {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-good": {{Kernel: kview("good")}}}}
l, h := judgingLeg(t, w)
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: 30 * time.Second})
if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot,health,kernel-good" {
t.Fatalf("calls = %s", got)
}
if r.Outcome != "applied" || !r.Healthy {
t.Fatalf("report = %+v", r)
}
if len(h.reports) != 2 || h.reports[0].Outcome != "judging" || h.reports[1].Outcome != "applied" {
t.Fatalf("hub reports = %+v", h.reports)
}
if w.plans[1]["vmid"] != float64(9201) {
t.Fatalf("the health reading must use the step's own guest, got %v", w.plans[1]["vmid"])
}
}
// An unhealthy one-shot boot: wait the full judge time, tell the hub, then ONE kernel-revert.
// COMPANION RED-PROOF (observed): return before the kernel-revert call in judgeKernel → "calls" fails.
func TestKernelAfterBoot_UnhealthyRevertsOnceAfterTheWait(t *testing.T) {
down := hostOK()
down.GuestRunning = new(bool)
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"health": {{Health: down}},
"kernel-revert": {{Kernel: kview("reverting")}}}}
l, h := judgingLeg(t, w)
start := l.now()
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: time.Minute})
calls := kernelCalls(w)
if calls[len(calls)-1] != "kernel-revert" || strings.Count(strings.Join(calls, ","), "kernel-revert") != 1 {
t.Fatalf("calls = %v", calls)
}
if waited := l.now().Sub(start); waited < 10*time.Minute {
t.Fatalf("reverted after %s — before the judge wait", waited)
}
if r.Outcome != "health_failed" || !strings.Contains(r.HealthReason, "not running") {
t.Fatalf("report = %+v", r)
}
if last := h.reports[len(h.reports)-1]; last.Outcome != "health_failed" {
t.Fatalf("the hub must hear health_failed before the revert reboot: %+v", h.reports)
}
for _, p := range w.plans {
if p["mode"] == "kernel-revert" && !strings.Contains(p["reason"].(string), "not running") {
t.Fatalf("the revert must carry the reason: %v", p)
}
}
}
// A box that never reaches the hub is not "healthy" — it reverts too.
func TestKernelAfterBoot_NoHubMeansRevert(t *testing.T) {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-revert": {{Kernel: kview("reverting")}}}}
l, _ := judgingLeg(t, w)
l.Hub = unreachableHub{}
l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 5 * time.Minute, Poll: time.Minute})
if c := kernelCalls(w); c[len(c)-1] != "kernel-revert" {
t.Fatalf("calls = %v", c)
}
}
type unreachableHub struct{}
func (unreachableHub) PostOSReport(context.Context, []byte) error { return context.DeadlineExceeded }
// What kernel-boot found becomes the hub's outcome, with no judging and no reboot.
func TestKernelAfterBoot_FallBackAndRevertResultsAreReported(t *testing.T) {
for ev, want := range map[string]string{"fell_back": "fell_back", "self_reverted": "self_reverted", "revert_failed": "revert_failed"} {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: ev,
Kernel: mustRaw(KernelView{Running: kOld, Default: kOld, Phase: ev, From: kOld, To: kNew, Reason: "r", VMID: 9201})}}}}
l, h := judgingLeg(t, w)
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{})
if r.Outcome != want || len(h.reports) != 1 || h.reports[0].Outcome != want {
t.Fatalf("%s: report %+v hub %+v", ev, r, h.reports)
}
if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot" {
t.Fatalf("%s: calls = %s", ev, got)
}
}
// nothing to do → nothing reported
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: "none", Kernel: kview("good")}}}}
l, h := judgingLeg(t, w)
if r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{}); r.Layer != "" || len(h.reports) != 0 {
t.Fatalf("an ordinary boot must report nothing: %+v %+v", r, h.reports)
}
}
// The signed executor STAGES (listed + the raw envelope + the kver) and never reboots.
func TestKernelStepExecutor_StagesNeverReboots(t *testing.T) {
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0",
Version: "7.0.14-22"}}, Authority: "signed", Kernel: kview("staged")}}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
e := KernelStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
params, _ := json.Marshal(KernelStepParams{ReleaseID: "os-kernel-1", Kver: kNew,
Packages: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22", Origin: PVEOrigin}}})
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_kernel_step"}`), Sig: []byte("SIG")})
if err := e.Execute(ctx, OpKernelStep, params); err != nil {
t.Fatal(err)
}
pp := w.plans[len(w.plans)-1]
sg, _ := pp["signed"].(map[string]any)
if pp["mode"] != "apply" || pp["select"] != "listed" || pp["expect_kver"] != kNew || sg == nil ||
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_kernel_step"}`)) {
t.Fatalf("plan = %v", pp)
}
if got := strings.Join(kernelCalls(w), ","); got != "apply" {
t.Fatalf("a signed stage must never reboot: %s", got)
}
if len(h.reports) != 1 || h.reports[0].Outcome != "staged" {
t.Fatalf("hub = %+v", h.reports)
}
if err := e.Execute(context.Background(), OpKernelStep, params); err == nil {
t.Fatal("no envelope must refuse")
}
bad, _ := json.Marshal(KernelStepParams{Kver: "x", Packages: []Package{{Name: "a"}}})
if err := e.Execute(ctx, OpKernelStep, bad); err == nil {
t.Fatal("a bad kver must refuse")
}
if err := e.Execute(ctx, OpPVEStep, params); err != signedjobs.ErrNoExecutor {
t.Fatalf("another op must pass through the chain: %v", err)
}
}
// os_kernel_step is never benign.
func TestKernelStep_IsDestructiveClass(t *testing.T) {
if reconcile.Classify(reconcile.ClassOSKernelStep, reconcile.Provenance{}) != reconcile.Destructive {
t.Fatal("os_kernel_step must be destructive-class (signed, operational key)")
}
}
// A kept stage report (the agent was killed mid-stage) reaches the hub as "staged" with its kernel view.
func TestKernel_KeptStageReportIsStaged(t *testing.T) {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, tonight(0))
ring := 0
rep := l.reportFromKept(context.Background(), WrapperReport{Layer: LayerKernel, Mode: "apply", Ring: &ring,
Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Kernel: kview("staged")}, "/x/report-r-kernel-apply.json")
if rep.Outcome != "staged" || !rep.Healthy || len(rep.Kernel) == 0 {
t.Fatalf("kept = %+v", rep)
}
}
+23 -1
View File
@@ -45,6 +45,9 @@ const (
// LayerPVE is the HOST's Proxmox userspace packages — slow lane (R-812 option A, `09` §3 decision 163, `11` §5.10):
// ring 0 in the night leg after a healthy host step, ring 1 only inside a signed os_pve_step. Never a kernel.
LayerPVE = "pve"
// LayerKernel is the HOST's kernel — the kernel lane (R-836, `09` §3 decision 172, `11` §5.11): a one-shot boot of
// the new kernel through the ESP flag, the default moved only after a healthy boot (kernel.go).
LayerKernel = "kernel"
)
// PVEOrigin is the origin apt prints for download.proxmox.com (the wrapper's PVE_ORIGIN); InstalledPVEOrigin is how
@@ -126,6 +129,11 @@ type WrapperReport struct {
OOMCheck json.RawMessage `json:"oom_check"`
// PVEManager: the pve layer — pveversion's pve-manager version after the step ("unknown" = unreadable).
PVEManager string `json:"pve_manager"`
// Kernel (R-836): the kernel layer's view {running, default, flag, phase, from, to, …}; KernelEvent what kernel-boot
// found after a boot; OutcomeHint "nothing" when no kernel was pending.
Kernel json.RawMessage `json:"kernel"`
KernelEvent string `json:"kernel_event"`
OutcomeHint string `json:"outcome_hint"`
// R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the
// agent process that started the pass. ReleaseID / VMID were always in the report.
RunID string `json:"run_id"`
@@ -168,6 +176,10 @@ type Report struct {
OOMCheck json.RawMessage `json:"oom_check,omitempty"`
// PVEManager: pve layer — pve-manager's version after the step (the hub's System page; R-812 option A).
PVEManager string `json:"pve_manager,omitempty"`
// Kernel: kernel layer — the wrapper's kernel view, byte for byte (running, default, flag, phase, from, to). The
// outcomes of this layer: staged | applied (the new kernel is the default) | fell_back | health_failed (self-revert
// started) | self_reverted | revert_failed | judging | nothing | refused | failed.
Kernel json.RawMessage `json:"kernel,omitempty"`
unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it
}
@@ -500,7 +512,7 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
// Pass is one leg's reports; an empty Layer means the step did not run.
type Pass struct {
Guest, Host, Docker, PVE Report
Guest, Host, Docker, PVE, Kernel Report
}
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
@@ -542,6 +554,16 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
default:
p.PVE = l.runPVE(ctx, g.RunID, vmid, trigger, blk, dockerOpts{})
}
// R-836 (`09` §3 decision 172): the kernel step ENDS the night — an appliance, after a healthy host step (and a
// healthy Proxmox step when one ran), only on a night the hub marks as told. It reboots the box. Pinned by TestKernel_*.
switch {
case !l.Appliance || h.Layer == "" || !okStep(h):
lg.Info("osupdate: kernel step skipped — no healthy host step this pass (an appliance only)", "appliance", l.Appliance, "host_outcome", h.Outcome)
case p.PVE.Layer != "" && !okStep(p.PVE):
lg.Warn("osupdate: kernel step skipped — the Proxmox step did not end healthy", "pve_outcome", p.PVE.Outcome)
default:
p.Kernel = l.runKernel(ctx, g.RunID, vmid, trigger, blk)
}
return p
}
+12 -1
View File
@@ -25,6 +25,7 @@ type fakeWrapper struct {
plans []map[string]any
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
pveGateHeld bool
kernelRep map[string][]WrapperReport // R-836: per kernel-layer mode, successive answers (the last one repeats)
}
func yes() *bool { b := true; return &b }
@@ -52,9 +53,19 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
f.plans = append(f.plans, plan)
layer := plan["layer"].(string)
ok := guestOK()
if layer == LayerHost || layer == LayerPVE {
if layer == LayerHost || layer == LayerPVE || layer == LayerKernel {
ok = hostOK()
}
if layer == LayerKernel {
if seq := f.kernelRep[plan["mode"].(string)]; len(seq) > 0 {
rep := seq[0]
if len(seq) > 1 {
f.kernelRep[plan["mode"].(string)] = seq[1:]
}
out, _ := json.Marshal(rep)
return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil
}
}
if layer == LayerPVE && plan["mode"] == "apply" {
f.pveGateHeld = pvegate.Stepping() // R-812: the /etc/pve write gate must be held while the pve step runs
}
+11
View File
@@ -142,6 +142,17 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
default:
rep.Outcome = "applied"
}
if wr.Layer == LayerKernel {
// a kernel STAGE changes nothing the box runs (the new kernel only boots once, at the night's reboot), so its
// kept copy needs no fresh health reading (R-836)
rep.Kernel = rawOrNil(wr.Kernel)
rep.Upgraded, rep.PassSeconds, rep.Authority = wr.Upgraded, wr.PassSeconds, wr.Authority
if rep.Outcome == "applied" {
rep.Outcome = "staged"
}
rep.Healthy, rep.HealthReason = rep.Outcome == "staged" || rep.Outcome == "nothing", prefix
return rep
}
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
rep.OOMCheck = rawOrNil(wr.OOMCheck)
+6 -1
View File
@@ -56,6 +56,11 @@ const (
// key) like os_docker_step; the root wrapper re-verifies the same signature itself.
ClassOSPVEStep OpClass = "os_pve_step"
// A kernel step on the host (R-836, `09` §3 decision 172, `11` §5.11) — ring 1: it STAGES a kernel (install + the
// one-shot flag; the night leg reboots it). Destructive-class (signed, operational key) like os_pve_step; the root
// wrapper re-verifies the same signature itself.
ClassOSKernelStep OpClass = "os_kernel_step"
// The config bundle (agent v0.143.0, R-840, `11` §5.4.2): the box's ROOT-OWNED files (sudoers, wrappers, units).
// Destructive-class (signed, operational key) like agent_update; the root wrapper re-verifies the signature itself.
ClassAgentConfigUpdate OpClass = "agent_config_update"
@@ -127,7 +132,7 @@ func Classify(class OpClass, prov Provenance) Disposition {
return Destructive
case ClassKeyRotation:
return Destructive
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassAgentConfigUpdate:
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassOSKernelStep, ClassAgentConfigUpdate:
// Never benign — no agent-internal provenance can make replacing the agent binary
// unsigned-safe (a compromised process must not be able to self-bless an update).
return Destructive