R-836: the kernel lane — one-shot boot through the ESP flag, boot good / one self-revert, night step on a told night
gates / gates (push) Successful in 44s
gates / gates (push) Successful in 44s
Wrapper layer kernel (stage / reboot / boot / good / revert / cancel / status; R20-R23), the two GRUB generators in the bundle (option C on the one-shot entry), the agent's night step and after-boot judge (host health rule + hub reached, 20 min measured), the signed os_kernel_step (stage only). Red-proofs: felhom.eu audits/kernel-lane-2026-10-07/A/redproof.txt. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -636,6 +636,17 @@ type WireOSUpdate struct {
|
||||
// HostRelease is the newest approved HOST release (hub v0.131.0, `11` §8 step 3) — a separate set: a version
|
||||
// approved for the guest is not approved for the host by that fact alone.
|
||||
HostRelease *WireOSRelease `json:"host_release,omitempty"`
|
||||
// Kernel is the kernel lane's instruction for THIS box (R-836, `09` §3 decision 172, `11` §5.11): the kernel the
|
||||
// household was told about, and whether tonight is a told night. Nil (an older hub, or no kernel due) = no kernel
|
||||
// step. The hub sets Tonight only after the household's mail the day before went out — no mail, no step.
|
||||
Kernel *WireKernelStep `json:"kernel,omitempty"`
|
||||
}
|
||||
|
||||
// WireKernelStep is the hub's kernel-lane instruction (hub osupdates.KernelBlock — field-exact, cross-repo).
|
||||
type WireKernelStep struct {
|
||||
Kver string `json:"kver"` // e.g. "7.0.14-22-pve" — the wrapper refuses any other (R23)
|
||||
Tonight bool `json:"tonight"` // the household was mailed the day before: tonight's leg may reboot
|
||||
NotifiedAt string `json:"notified_at,omitempty"` // when that mail went out (RFC 3339), for the log
|
||||
}
|
||||
|
||||
// WireOSRelease is an approved version set; Snapshot is the approval time (YYYYMMDDTHHMMSSZ) the wrapper uses
|
||||
|
||||
@@ -0,0 +1,409 @@
|
||||
package osupdate
|
||||
|
||||
// The kernel lane (R-836, `09` §3 decisions 164 + 172, `11` §5.11). The root half is felhom-os-apply's layer "kernel"
|
||||
// (configs/, its own tests); this file decides WHEN and judges the boot:
|
||||
//
|
||||
// - the night leg (Run → runKernel): after a healthy host step, on a night the hub marks as told (the household was
|
||||
// mailed the day before — no mail, no step): ring 0 STAGES the pending kernel (select pending-kernel, the root-owned
|
||||
// ring-0 mark) and reboots; ring 1 reboots only a step a signed os_kernel_step staged earlier (KernelStepExecutor).
|
||||
// - after a boot (KernelAfterBoot, at daemon start): the wrapper says what became of the step. On the new kernel the
|
||||
// agent JUDGES the boot — the host health rule (`11` §8.2: the Proxmox daemons, the guest running and healthy, the
|
||||
// tunnel) AND the box reaching the hub — for KernelJudgeWait. Healthy → kernel-good (the new kernel becomes the
|
||||
// default). Not healthy by the deadline → ONE self-revert (kernel-revert: a reboot into the old kernel, still the
|
||||
// default). A crash on the new kernel needs nothing from the agent: GRUB already boots the old default.
|
||||
//
|
||||
// The host is rebooted by this file only through the wrapper (kernel-reboot, kernel-revert), and only for a staged step.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||
)
|
||||
|
||||
// OpKernelStep is the signed op class that STAGES a kernel set on a ring-1 box (it never reboots: the night leg does,
|
||||
// once the household was told). CC may sign it until the first paying customer (R-530 ruling).
|
||||
const OpKernelStep = "os_kernel_step"
|
||||
|
||||
// DefaultKernelJudgeWait is how long a one-shot boot may take to come back healthy before the agent reverts it ONCE.
|
||||
// Measured 2026-10-07 (`audits/kernel-lane-2026-10-07/B/`): every container healthy 68 s after a reboot on demo-felhom,
|
||||
// 272 s on demo-hp; 20 minutes stays under the hub's 30-minute host_stale (a box that never comes back alarms after it).
|
||||
const DefaultKernelJudgeWait = 20 * time.Minute
|
||||
|
||||
var kverRE = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+-[0-9]+-pve$`)
|
||||
|
||||
// KernelView is the wrapper's kernel object (felhom-os-apply Kernel.view).
|
||||
type KernelView struct {
|
||||
Running string `json:"running"`
|
||||
Default string `json:"default"`
|
||||
Flag *string `json:"flag"`
|
||||
Phase string `json:"phase"`
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
SelfRevertUsed bool `json:"self_revert_used"`
|
||||
Reason string `json:"reason"`
|
||||
VMID int `json:"vmid"` // the customer guest the step was staged for (the health rule's guest)
|
||||
}
|
||||
|
||||
func parseKernel(raw json.RawMessage) KernelView {
|
||||
var v KernelView
|
||||
_ = json.Unmarshal(raw, &v)
|
||||
return v
|
||||
}
|
||||
|
||||
// kernelPlan is one kernel-layer wrapper call.
|
||||
func kernelPlan(mode string, vmid int, extra map[string]any) map[string]any {
|
||||
p := map[string]any{"release_id": "kernel", "layer": LayerKernel, "lane": "slow", "vmid": vmid, "mode": mode,
|
||||
"packages": []Package{}}
|
||||
for k, v := range extra {
|
||||
p[k] = v
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// KernelStatus reads the kernel lane's state (read only).
|
||||
func (l *Leg) KernelStatus(ctx context.Context, vmid int) (KernelView, error) {
|
||||
wr, err := l.call(ctx, "kstatus"+l.now().UTC().Format("150405"), kernelPlan("kernel-status", vmid, nil))
|
||||
if err != nil {
|
||||
return KernelView{}, err
|
||||
}
|
||||
if wr.refused() {
|
||||
return KernelView{}, fmt.Errorf("kernel-status refused: %s", wr.Refused)
|
||||
}
|
||||
return parseKernel(wr.Kernel), nil
|
||||
}
|
||||
|
||||
// kernelDue reports whether tonight's leg may take a kernel step, and why not.
|
||||
func kernelDue(blk hub.WireOSUpdate, trigger string) (bool, string) {
|
||||
switch {
|
||||
case trigger != "night":
|
||||
return false, "a kernel step runs only in the night leg (never a debug pass)"
|
||||
case blk.Kernel == nil:
|
||||
return false, "the hub names no kernel step for this box"
|
||||
case !blk.Enabled:
|
||||
return false, "OS updates are switched off for this box"
|
||||
case !kverRE.MatchString(blk.Kernel.Kver):
|
||||
return false, "the hub's kernel " + blk.Kernel.Kver + " is not a kernel version"
|
||||
case !blk.Kernel.Tonight:
|
||||
return false, "the household has not been told about tonight (no mail, no step — `09` §3 decision 172)"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// runKernel is the night leg's last step. It returns the stage report (Layer "" when nothing ran). On success the box
|
||||
// is rebooting when it returns.
|
||||
func (l *Leg) runKernel(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
|
||||
lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", trigger, "ring", blk.Ring)
|
||||
if ok, why := kernelDue(blk, trigger); !ok {
|
||||
lg.Info("osupdate: kernel step skipped — " + why)
|
||||
return Report{}
|
||||
}
|
||||
want := blk.Kernel.Kver
|
||||
st, err := l.KernelStatus(ctx, vmid)
|
||||
if err != nil {
|
||||
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
|
||||
Mode: "apply", Outcome: "failed", HealthReason: "kernel status unreadable: " + err.Error()})
|
||||
}
|
||||
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply", ReleaseID: want}
|
||||
switch {
|
||||
case st.Phase == "staged" && st.To == want:
|
||||
lg.Info("osupdate: kernel step — a staged kernel waits for tonight", "from", st.From, "to", st.To)
|
||||
rep.Outcome, rep.Healthy = "staged", true
|
||||
case blk.Ring != 0:
|
||||
lg.Info("osupdate: kernel step skipped — ring 1 boots only a kernel a signed os_kernel_step staged", "phase", st.Phase, "staged", st.To, "want", want)
|
||||
return Report{}
|
||||
default:
|
||||
wr, cerr := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": "ring0-" + runID,
|
||||
"select": "pending-kernel", "expect_kver": want, "run_id": runID, "trigger": trigger, "ring": blk.Ring}))
|
||||
rep.unsent = reportFile(l.planDir(), runID, LayerKernel, "apply")
|
||||
rep.Kernel = rawOrNil(wr.Kernel)
|
||||
switch {
|
||||
case cerr != nil:
|
||||
rep.Outcome, rep.HealthReason = "failed", cerr.Error()
|
||||
return l.finish(ctx, lg, rep)
|
||||
case wr.refused():
|
||||
rep.Outcome, rep.Refused = "refused", wr.Refused
|
||||
return l.finish(ctx, lg, rep)
|
||||
case wr.failed():
|
||||
rep.Outcome, rep.Refused = "failed", wr.Failed
|
||||
return l.finish(ctx, lg, rep)
|
||||
case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0:
|
||||
rep.Outcome, rep.Healthy = "nothing", true
|
||||
return l.finish(ctx, lg, rep)
|
||||
}
|
||||
rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds
|
||||
rep.RebootNeeded = true
|
||||
}
|
||||
rep = l.finish(ctx, lg, rep) // the hub hears "staged" BEFORE the box goes down
|
||||
wr, err := l.call(ctx, runID, kernelPlan("kernel-reboot", vmid, nil))
|
||||
switch {
|
||||
case err != nil:
|
||||
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
|
||||
Mode: "kernel-reboot", ReleaseID: want, Outcome: "failed", HealthReason: "kernel-reboot: " + err.Error()})
|
||||
case wr.refused() || wr.failed():
|
||||
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: trigger, Ring: blk.Ring, VMID: vmid,
|
||||
Mode: "kernel-reboot", ReleaseID: want, Outcome: "refused", Refused: firstRaw(wr.Refused, wr.Failed),
|
||||
Kernel: rawOrNil(wr.Kernel)})
|
||||
}
|
||||
lg.Warn("osupdate: kernel step — the box restarts now for its one-shot boot", "to", want)
|
||||
return rep
|
||||
}
|
||||
|
||||
func firstRaw(a, b json.RawMessage) json.RawMessage {
|
||||
if r := rawOrNil(a); r != nil {
|
||||
return r
|
||||
}
|
||||
return rawOrNil(b)
|
||||
}
|
||||
|
||||
// KernelJudge is what KernelAfterBoot needs besides the leg: the hub reachability probe is the "judging" report itself.
|
||||
type KernelJudge struct {
|
||||
Wait time.Duration // default DefaultKernelJudgeWait
|
||||
Poll time.Duration // default 30 s
|
||||
}
|
||||
|
||||
// KernelAfterBoot runs once at daemon start: what became of a kernel step across the boot. On the new kernel it judges
|
||||
// the boot (blocking up to the wait — run it in a goroutine). vmid 0 = the guest the step recorded (it may not run yet).
|
||||
func (l *Leg) KernelAfterBoot(ctx context.Context, vmid int, j KernelJudge) Report {
|
||||
runID := "boot-" + l.now().UTC().Format("20060102T150405Z")
|
||||
lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid)
|
||||
wr, err := l.call(ctx, runID, kernelPlan("kernel-boot", vmid, nil))
|
||||
if err != nil {
|
||||
lg.Warn("osupdate: kernel after-boot check failed", "err", err)
|
||||
return Report{}
|
||||
}
|
||||
if wr.refused() {
|
||||
lg.Info("osupdate: kernel after-boot check refused (an older wrapper, or a BYO host)", "refused", string(wr.Refused))
|
||||
return Report{}
|
||||
}
|
||||
v := parseKernel(wr.Kernel)
|
||||
if vmid <= 0 {
|
||||
vmid = v.VMID // after a boot the guest may not run yet — the step's own record names it
|
||||
}
|
||||
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid, Mode: "kernel-boot",
|
||||
ReleaseID: v.To, Kernel: rawOrNil(wr.Kernel)}
|
||||
switch wr.KernelEvent {
|
||||
case "fell_back":
|
||||
rep.Outcome, rep.HealthReason = "fell_back", v.Reason
|
||||
lg.Warn("osupdate: kernel step FELL BACK — the new kernel did not come up; the box runs the old one", "from", v.From, "to", v.To)
|
||||
return l.finish(ctx, lg, rep)
|
||||
case "self_reverted":
|
||||
rep.Outcome, rep.HealthReason = "self_reverted", v.Reason
|
||||
lg.Warn("osupdate: kernel step SELF-REVERTED — back on the old kernel", "from", v.From, "to", v.To, "reason", v.Reason)
|
||||
return l.finish(ctx, lg, rep)
|
||||
case "revert_failed":
|
||||
rep.Outcome, rep.HealthReason = "revert_failed", v.Reason
|
||||
lg.Error("osupdate: kernel self-revert came back on the NEW kernel — no second revert; the operator decides", "to", v.To)
|
||||
return l.finish(ctx, lg, rep)
|
||||
case "judging":
|
||||
return l.judgeKernel(ctx, runID, vmid, v, wr.HealthBefore, j, lg)
|
||||
}
|
||||
return Report{}
|
||||
}
|
||||
|
||||
// KernelVerdict is THE one-shot boot rule (R-836; pinned by TestKernelVerdict): the host health rule (`11` §8.2 —
|
||||
// the Proxmox daemons and the agent active, the customer guest running and its own rule passing, the tunnel running)
|
||||
// AND the box reached the hub since this boot.
|
||||
func KernelVerdict(before, after *Health, tunnel string, hubReached bool) (bool, string) {
|
||||
if ok, why := HostHealthVerdict(before, after, tunnel); !ok {
|
||||
return false, why
|
||||
}
|
||||
if !hubReached {
|
||||
return false, "the box has not reached the hub since the boot"
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
func (l *Leg) judgeKernel(ctx context.Context, runID string, vmid int, v KernelView, before *Health, j KernelJudge, lg *slog.Logger) Report {
|
||||
wait, poll := j.Wait, j.Poll
|
||||
if wait <= 0 {
|
||||
wait = DefaultKernelJudgeWait
|
||||
}
|
||||
if poll <= 0 {
|
||||
poll = 30 * time.Second
|
||||
}
|
||||
lg.Info("osupdate: kernel step — judging the one-shot boot", "from", v.From, "to", v.To, "wait", wait.String())
|
||||
start := l.now()
|
||||
deadline := start.Add(wait)
|
||||
hubReached := false
|
||||
var why string
|
||||
for {
|
||||
if !hubReached && l.Hub != nil {
|
||||
// the hub's reachability IS this report reaching it (and the operator sees the box is back on the new kernel)
|
||||
body, _ := json.Marshal(Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
|
||||
Mode: "kernel-boot", ReleaseID: v.To, Outcome: "judging", Kernel: mustRaw(v)})
|
||||
rctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
if err := l.Hub.PostOSReport(rctx, body); err == nil {
|
||||
hubReached = true
|
||||
lg.Info("osupdate: kernel step — the box reached the hub on the new kernel", "after", l.now().Sub(start).Round(time.Second).String())
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
var h *Health
|
||||
hr, err := l.call(ctx, runID, kernelPlan("health", vmid, nil))
|
||||
switch {
|
||||
case err != nil:
|
||||
why = "no health reading: " + err.Error()
|
||||
case hr.refused():
|
||||
why = "no health reading: " + string(hr.Refused)
|
||||
default:
|
||||
h = hr.Health
|
||||
}
|
||||
if h != nil {
|
||||
t := hub.TunnelUnknown
|
||||
if l.Tunnel != nil {
|
||||
t, _ = l.Tunnel.Status(ctx)
|
||||
}
|
||||
var ok bool
|
||||
ok, why = KernelVerdict(before, h, t, hubReached)
|
||||
if ok {
|
||||
return l.kernelGood(ctx, runID, vmid, v, start, lg)
|
||||
}
|
||||
}
|
||||
if !l.now().Before(deadline) || ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
l.sleep(ctx, poll)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
lg.Warn("osupdate: kernel judging stopped (the agent is stopping) — the next start judges again", "reason", why)
|
||||
return Report{}
|
||||
}
|
||||
// not healthy by the deadline: tell the hub (best effort), then ONE self-revert into the old kernel
|
||||
rep := l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
|
||||
Mode: "kernel-revert", ReleaseID: v.To, Outcome: "health_failed", HealthReason: why + " — reverting to " + v.From,
|
||||
Kernel: mustRaw(v)})
|
||||
lg.Error("osupdate: kernel step — the one-shot boot is NOT healthy; restarting ONCE into the old kernel", "reason", why,
|
||||
"waited", wait.String(), "from", v.From, "to", v.To)
|
||||
wr, err := l.call(ctx, runID, kernelPlan("kernel-revert", vmid, map[string]any{"reason": truncate(why, 280)}))
|
||||
if err != nil || wr.refused() || wr.failed() {
|
||||
lg.Error("osupdate: kernel self-revert did not start — the box stays on the new kernel; the operator decides",
|
||||
"err", err, "refused", string(firstRaw(wr.Refused, wr.Failed)))
|
||||
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid,
|
||||
Mode: "kernel-revert", ReleaseID: v.To, Outcome: "revert_failed", Refused: firstRaw(wr.Refused, wr.Failed),
|
||||
HealthReason: "the self-revert did not start"})
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
func (l *Leg) kernelGood(ctx context.Context, runID string, vmid int, v KernelView, start time.Time, lg *slog.Logger) Report {
|
||||
wr, err := l.call(ctx, runID, kernelPlan("kernel-good", vmid, nil))
|
||||
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "boot", Ring: l.Block().Ring, VMID: vmid, Mode: "kernel-good",
|
||||
ReleaseID: v.To}
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.Outcome, rep.HealthReason = "failed", "kernel-good: "+err.Error()
|
||||
case wr.refused() || wr.failed():
|
||||
rep.Outcome, rep.Refused, rep.HealthReason = "failed", firstRaw(wr.Refused, wr.Failed), "kernel-good did not move the default"
|
||||
default:
|
||||
rep.Outcome, rep.Healthy = "applied", true
|
||||
rep.HealthReason = fmt.Sprintf("healthy %s after the agent started; the new kernel is the default", l.now().Sub(start).Round(time.Second))
|
||||
}
|
||||
rep.Kernel = rawOrNil(wr.Kernel)
|
||||
lg.Info("osupdate: kernel step — "+rep.Outcome, "to", v.To, "reason", rep.HealthReason)
|
||||
return l.finish(ctx, lg, rep)
|
||||
}
|
||||
|
||||
func mustRaw(v any) json.RawMessage {
|
||||
b, _ := json.Marshal(v)
|
||||
return b
|
||||
}
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n]
|
||||
}
|
||||
|
||||
// KernelStepParams are a signed os_kernel_step's params: the exact kernel set (the wrapper compares it with the plan).
|
||||
type KernelStepParams struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Packages []Package `json:"packages"`
|
||||
Kver string `json:"kver"`
|
||||
VMID int `json:"vmid,omitempty"`
|
||||
}
|
||||
|
||||
// KernelStepExecutor STAGES a verified os_kernel_step (signedjobs.Executor) under the heavy-op gate. It never reboots:
|
||||
// the night leg reboots a staged kernel on a night the household was told about.
|
||||
type KernelStepExecutor struct {
|
||||
Leg *Leg
|
||||
Guest func(ctx context.Context) (int, error)
|
||||
Gate func(ctx context.Context) (release func(), err error)
|
||||
}
|
||||
|
||||
// Execute implements signedjobs.Executor.
|
||||
func (e KernelStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
|
||||
if op != OpKernelStep {
|
||||
return signedjobs.ErrNoExecutor
|
||||
}
|
||||
so, ok := signedjobs.SignedOpFrom(ctx)
|
||||
if !ok {
|
||||
return fmt.Errorf("os_kernel_step: no signed envelope in the context — the wrapper could not verify it")
|
||||
}
|
||||
var p KernelStepParams
|
||||
if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 || !kverRE.MatchString(p.Kver) {
|
||||
return fmt.Errorf("os_kernel_step: params must name the kernel set and its kver: %v", err)
|
||||
}
|
||||
vmid := p.VMID
|
||||
if vmid == 0 {
|
||||
if e.Guest == nil {
|
||||
return fmt.Errorf("os_kernel_step: no vmid and no guest finder")
|
||||
}
|
||||
v, err := e.Guest(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("os_kernel_step: find the customer guest: %w", err)
|
||||
}
|
||||
vmid = v
|
||||
}
|
||||
if e.Gate != nil {
|
||||
release, err := e.Gate(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("os_kernel_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
|
||||
}
|
||||
defer release()
|
||||
}
|
||||
rep := e.Leg.StageKernelSigned(ctx, vmid, p, so.Blob, string(so.Sig))
|
||||
if rep.Outcome == "staged" || rep.Outcome == "nothing" {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("os_kernel_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
|
||||
}
|
||||
|
||||
// StageKernelSigned stages a signed kernel set (ring 1): install + flag, no reboot.
|
||||
func (l *Leg) StageKernelSigned(ctx context.Context, vmid int, p KernelStepParams, blob []byte, sig string) Report {
|
||||
unlock := l.lockPass(true)
|
||||
defer unlock()
|
||||
l.sendUnsentLocked(ctx) // R-868
|
||||
runID := l.now().UTC().Format("20060102T150405Z")
|
||||
rid := p.ReleaseID
|
||||
if rid == "" {
|
||||
rid = "signed-" + runID
|
||||
}
|
||||
lg := l.log().With("run", runID, "layer", LayerKernel, "vmid", vmid, "trigger", "signed", "release", rid)
|
||||
wr, err := l.call(ctx, runID, kernelPlan("apply", vmid, map[string]any{"release_id": rid, "select": "listed",
|
||||
"packages": p.Packages, "expect_kver": p.Kver, "run_id": runID, "trigger": "signed", "ring": l.Block().Ring,
|
||||
"signed": map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}}))
|
||||
rep := Report{RunID: runID, Layer: LayerKernel, Trigger: "signed", Ring: l.Block().Ring, VMID: vmid, Mode: "apply",
|
||||
ReleaseID: rid, Kernel: rawOrNil(wr.Kernel), unsent: reportFile(l.planDir(), runID, LayerKernel, "apply")}
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.Outcome, rep.HealthReason = "failed", err.Error()
|
||||
case wr.refused():
|
||||
rep.Outcome, rep.Refused = "refused", wr.Refused
|
||||
case wr.failed():
|
||||
rep.Outcome, rep.Refused = "failed", wr.Failed
|
||||
case wr.OutcomeHint == "nothing" || len(wr.Upgraded) == 0:
|
||||
rep.Outcome, rep.Healthy = "nothing", true
|
||||
default:
|
||||
rep.Outcome, rep.Healthy, rep.Upgraded, rep.Authority, rep.PassSeconds = "staged", true, wr.Upgraded, wr.Authority, wr.PassSeconds
|
||||
rep.RebootNeeded = true
|
||||
}
|
||||
return l.finish(ctx, lg, rep)
|
||||
}
|
||||
@@ -0,0 +1,314 @@
|
||||
package osupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||
)
|
||||
|
||||
// ---- the kernel lane (R-836, `09` §3 decision 172, `11` §5.11) ----
|
||||
|
||||
const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve"
|
||||
|
||||
func kview(phase string) json.RawMessage {
|
||||
return mustRaw(KernelView{Running: kOld, Default: kOld, Phase: phase, From: kOld, To: kNew, VMID: 9201})
|
||||
}
|
||||
|
||||
func tonight(ring int) *hub.WireOSUpdate {
|
||||
return &hub.WireOSUpdate{Ring: ring, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}
|
||||
}
|
||||
|
||||
func kernelCalls(w *fakeWrapper) []string {
|
||||
var m []string
|
||||
for _, p := range w.plans {
|
||||
if p["layer"] == LayerKernel {
|
||||
m = append(m, p["mode"].(string))
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// Ring 0, a told night: after the healthy host step the leg stages the pending kernel (select pending-kernel, the
|
||||
// kernel the household was told about), tells the hub "staged", THEN reboots — the kernel step ends the night.
|
||||
func TestKernel_Ring0ToldNightStagesThenReboots(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{
|
||||
"kernel-status": {{Kernel: kview("none")}},
|
||||
"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}},
|
||||
"kernel-reboot": {{Kernel: kview("oneshot")}},
|
||||
}}
|
||||
l, h := newLeg(t, w, tonight(0))
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply,kernel-reboot" {
|
||||
t.Fatalf("kernel calls = %s", got)
|
||||
}
|
||||
var ap map[string]any
|
||||
for _, x := range w.plans {
|
||||
if x["layer"] == LayerKernel && x["mode"] == "apply" {
|
||||
ap = x
|
||||
}
|
||||
}
|
||||
if ap["select"] != "pending-kernel" || ap["expect_kver"] != kNew || ap["lane"] != "slow" {
|
||||
t.Fatalf("stage plan = %v", ap)
|
||||
}
|
||||
if p.Kernel.Outcome != "staged" || !p.Kernel.Healthy {
|
||||
t.Fatalf("kernel report = %+v", p.Kernel)
|
||||
}
|
||||
last := h.reports[len(h.reports)-1]
|
||||
if last.Layer != LayerKernel || last.Outcome != "staged" {
|
||||
t.Fatalf("the hub must hear 'staged' before the reboot: %+v", h.reports)
|
||||
}
|
||||
// the kernel step is the LAST wrapper call of the night
|
||||
if lp := w.plans[len(w.plans)-1]; lp["layer"] != LayerKernel || lp["mode"] != "kernel-reboot" {
|
||||
t.Fatalf("the reboot must end the night, last call = %v", lp)
|
||||
}
|
||||
}
|
||||
|
||||
// No mail, no step: a kernel the household was NOT told about never runs; nor in a debug pass; nor without a block.
|
||||
// COMPANION RED-PROOF (observed): drop the `!blk.Kernel.Tonight` case in kernelDue → the first sub-case fails.
|
||||
func TestKernel_NoMailNoStep(t *testing.T) {
|
||||
cases := map[string]struct {
|
||||
blk *hub.WireOSUpdate
|
||||
trigger string
|
||||
}{
|
||||
"not told": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: false}}, "night"},
|
||||
"debug pass": {tonight(0), "debug"},
|
||||
"no block": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night"},
|
||||
"switch off": {&hub.WireOSUpdate{Ring: 0, Enabled: false, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}, "night"},
|
||||
"bad kver": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: "7.0; reboot", Tonight: true}}, "night"},
|
||||
}
|
||||
for name, c := range cases {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, c.blk)
|
||||
p := l.Run(context.Background(), 9201, c.trigger)
|
||||
if len(kernelCalls(w)) != 0 || p.Kernel.Layer != "" {
|
||||
t.Fatalf("%s: a kernel step ran: %v", name, kernelCalls(w))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The kernel step needs a healthy host step on an appliance, and a healthy Proxmox step when one ran.
|
||||
func TestKernel_SkippedWithoutHealthyEarlierSteps(t *testing.T) {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, tonight(0))
|
||||
l.Appliance = false
|
||||
l.Run(context.Background(), 9201, "night")
|
||||
if len(kernelCalls(w)) != 0 {
|
||||
t.Fatalf("a BYO box took a kernel step: %v", kernelCalls(w))
|
||||
}
|
||||
w2 := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}},
|
||||
PVEManager: "9.2.2"}}} // pveversion still old → the pve step is unhealthy
|
||||
l2, _ := newLeg(t, w2, tonight(0))
|
||||
l2.Run(context.Background(), 9201, "night")
|
||||
if len(kernelCalls(w2)) != 0 {
|
||||
t.Fatalf("a kernel step ran after an unhealthy Proxmox step: %v", kernelCalls(w2))
|
||||
}
|
||||
}
|
||||
|
||||
// Ring 1 reboots only a kernel a signed job staged — never stages one itself in the night leg.
|
||||
func TestKernel_Ring1RebootsOnlyASignedStage(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}}}
|
||||
l, _ := newLeg(t, w, tonight(1))
|
||||
l.Run(context.Background(), 9201, "night")
|
||||
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status" {
|
||||
t.Fatalf("ring 1 without a staged kernel: calls = %s", got)
|
||||
}
|
||||
w2 := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("staged")}},
|
||||
"kernel-reboot": {{Kernel: kview("oneshot")}}}}
|
||||
l2, _ := newLeg(t, w2, tonight(1))
|
||||
p := l2.Run(context.Background(), 9201, "night")
|
||||
if got := strings.Join(kernelCalls(w2), ","); got != "kernel-status,kernel-reboot" || p.Kernel.Outcome != "staged" {
|
||||
t.Fatalf("ring 1 with a staged kernel: calls = %s report = %+v", got, p.Kernel)
|
||||
}
|
||||
}
|
||||
|
||||
// A refused stage never reboots.
|
||||
func TestKernel_RefusedStageNeverReboots(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}},
|
||||
"apply": {{Refused: json.RawMessage(`{"code":"R20","reason":"/boot/efi is not a mounted vfat ESP"}`)}}}}
|
||||
l, h := newLeg(t, w, tonight(0))
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply" || p.Kernel.Outcome != "refused" {
|
||||
t.Fatalf("calls = %s report = %+v", got, p.Kernel)
|
||||
}
|
||||
if last := h.reports[len(h.reports)-1]; last.Layer != LayerKernel || last.Outcome != "refused" {
|
||||
t.Fatalf("the hub must hear the refusal: %+v", last)
|
||||
}
|
||||
}
|
||||
|
||||
// THE one-shot boot rule: the host rule AND the hub reached. COMPANION RED-PROOF (observed): drop the hubReached
|
||||
// check in KernelVerdict → the second case fails.
|
||||
func TestKernelVerdict(t *testing.T) {
|
||||
if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, true); !ok {
|
||||
t.Fatalf("a healthy boot read unhealthy: %s", why)
|
||||
}
|
||||
if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, false); ok || !strings.Contains(why, "hub") {
|
||||
t.Fatalf("a box that has not reached the hub must not pass: ok=%v %q", ok, why)
|
||||
}
|
||||
down := hostOK()
|
||||
down.GuestRunning = new(bool)
|
||||
if ok, _ := KernelVerdict(hostOK(), down, hub.TunnelRunning, true); ok {
|
||||
t.Fatal("a guest that does not run must fail")
|
||||
}
|
||||
if ok, _ := KernelVerdict(hostOK(), hostOK(), hub.TunnelUnknown, true); ok {
|
||||
t.Fatal("an unknown tunnel must fail (the host rule)")
|
||||
}
|
||||
}
|
||||
|
||||
func judgingLeg(t *testing.T, w *fakeWrapper) (*Leg, *fakeHub) {
|
||||
if w.kernelRep == nil {
|
||||
w.kernelRep = map[string][]WrapperReport{}
|
||||
}
|
||||
if _, ok := w.kernelRep["kernel-boot"]; !ok {
|
||||
w.kernelRep["kernel-boot"] = []WrapperReport{{KernelEvent: "judging", Kernel: mustRaw(KernelView{Running: kNew,
|
||||
Default: kOld, Phase: "judging", From: kOld, To: kNew, VMID: 9201}), HealthBefore: hostOK()}}
|
||||
}
|
||||
return newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
}
|
||||
|
||||
// A healthy one-shot boot: the hub hears "judging", then kernel-good, then "applied".
|
||||
func TestKernelAfterBoot_HealthyBecomesTheDefault(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-good": {{Kernel: kview("good")}}}}
|
||||
l, h := judgingLeg(t, w)
|
||||
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: 30 * time.Second})
|
||||
if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot,health,kernel-good" {
|
||||
t.Fatalf("calls = %s", got)
|
||||
}
|
||||
if r.Outcome != "applied" || !r.Healthy {
|
||||
t.Fatalf("report = %+v", r)
|
||||
}
|
||||
if len(h.reports) != 2 || h.reports[0].Outcome != "judging" || h.reports[1].Outcome != "applied" {
|
||||
t.Fatalf("hub reports = %+v", h.reports)
|
||||
}
|
||||
if w.plans[1]["vmid"] != float64(9201) {
|
||||
t.Fatalf("the health reading must use the step's own guest, got %v", w.plans[1]["vmid"])
|
||||
}
|
||||
}
|
||||
|
||||
// An unhealthy one-shot boot: wait the full judge time, tell the hub, then ONE kernel-revert.
|
||||
// COMPANION RED-PROOF (observed): return before the kernel-revert call in judgeKernel → "calls" fails.
|
||||
func TestKernelAfterBoot_UnhealthyRevertsOnceAfterTheWait(t *testing.T) {
|
||||
down := hostOK()
|
||||
down.GuestRunning = new(bool)
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"health": {{Health: down}},
|
||||
"kernel-revert": {{Kernel: kview("reverting")}}}}
|
||||
l, h := judgingLeg(t, w)
|
||||
start := l.now()
|
||||
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: time.Minute})
|
||||
calls := kernelCalls(w)
|
||||
if calls[len(calls)-1] != "kernel-revert" || strings.Count(strings.Join(calls, ","), "kernel-revert") != 1 {
|
||||
t.Fatalf("calls = %v", calls)
|
||||
}
|
||||
if waited := l.now().Sub(start); waited < 10*time.Minute {
|
||||
t.Fatalf("reverted after %s — before the judge wait", waited)
|
||||
}
|
||||
if r.Outcome != "health_failed" || !strings.Contains(r.HealthReason, "not running") {
|
||||
t.Fatalf("report = %+v", r)
|
||||
}
|
||||
if last := h.reports[len(h.reports)-1]; last.Outcome != "health_failed" {
|
||||
t.Fatalf("the hub must hear health_failed before the revert reboot: %+v", h.reports)
|
||||
}
|
||||
for _, p := range w.plans {
|
||||
if p["mode"] == "kernel-revert" && !strings.Contains(p["reason"].(string), "not running") {
|
||||
t.Fatalf("the revert must carry the reason: %v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A box that never reaches the hub is not "healthy" — it reverts too.
|
||||
func TestKernelAfterBoot_NoHubMeansRevert(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-revert": {{Kernel: kview("reverting")}}}}
|
||||
l, _ := judgingLeg(t, w)
|
||||
l.Hub = unreachableHub{}
|
||||
l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 5 * time.Minute, Poll: time.Minute})
|
||||
if c := kernelCalls(w); c[len(c)-1] != "kernel-revert" {
|
||||
t.Fatalf("calls = %v", c)
|
||||
}
|
||||
}
|
||||
|
||||
type unreachableHub struct{}
|
||||
|
||||
func (unreachableHub) PostOSReport(context.Context, []byte) error { return context.DeadlineExceeded }
|
||||
|
||||
// What kernel-boot found becomes the hub's outcome, with no judging and no reboot.
|
||||
func TestKernelAfterBoot_FallBackAndRevertResultsAreReported(t *testing.T) {
|
||||
for ev, want := range map[string]string{"fell_back": "fell_back", "self_reverted": "self_reverted", "revert_failed": "revert_failed"} {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: ev,
|
||||
Kernel: mustRaw(KernelView{Running: kOld, Default: kOld, Phase: ev, From: kOld, To: kNew, Reason: "r", VMID: 9201})}}}}
|
||||
l, h := judgingLeg(t, w)
|
||||
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{})
|
||||
if r.Outcome != want || len(h.reports) != 1 || h.reports[0].Outcome != want {
|
||||
t.Fatalf("%s: report %+v hub %+v", ev, r, h.reports)
|
||||
}
|
||||
if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot" {
|
||||
t.Fatalf("%s: calls = %s", ev, got)
|
||||
}
|
||||
}
|
||||
// nothing to do → nothing reported
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: "none", Kernel: kview("good")}}}}
|
||||
l, h := judgingLeg(t, w)
|
||||
if r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{}); r.Layer != "" || len(h.reports) != 0 {
|
||||
t.Fatalf("an ordinary boot must report nothing: %+v %+v", r, h.reports)
|
||||
}
|
||||
}
|
||||
|
||||
// The signed executor STAGES (listed + the raw envelope + the kver) and never reboots.
|
||||
func TestKernelStepExecutor_StagesNeverReboots(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0",
|
||||
Version: "7.0.14-22"}}, Authority: "signed", Kernel: kview("staged")}}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
e := KernelStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
|
||||
params, _ := json.Marshal(KernelStepParams{ReleaseID: "os-kernel-1", Kver: kNew,
|
||||
Packages: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22", Origin: PVEOrigin}}})
|
||||
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_kernel_step"}`), Sig: []byte("SIG")})
|
||||
if err := e.Execute(ctx, OpKernelStep, params); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pp := w.plans[len(w.plans)-1]
|
||||
sg, _ := pp["signed"].(map[string]any)
|
||||
if pp["mode"] != "apply" || pp["select"] != "listed" || pp["expect_kver"] != kNew || sg == nil ||
|
||||
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_kernel_step"}`)) {
|
||||
t.Fatalf("plan = %v", pp)
|
||||
}
|
||||
if got := strings.Join(kernelCalls(w), ","); got != "apply" {
|
||||
t.Fatalf("a signed stage must never reboot: %s", got)
|
||||
}
|
||||
if len(h.reports) != 1 || h.reports[0].Outcome != "staged" {
|
||||
t.Fatalf("hub = %+v", h.reports)
|
||||
}
|
||||
if err := e.Execute(context.Background(), OpKernelStep, params); err == nil {
|
||||
t.Fatal("no envelope must refuse")
|
||||
}
|
||||
bad, _ := json.Marshal(KernelStepParams{Kver: "x", Packages: []Package{{Name: "a"}}})
|
||||
if err := e.Execute(ctx, OpKernelStep, bad); err == nil {
|
||||
t.Fatal("a bad kver must refuse")
|
||||
}
|
||||
if err := e.Execute(ctx, OpPVEStep, params); err != signedjobs.ErrNoExecutor {
|
||||
t.Fatalf("another op must pass through the chain: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// os_kernel_step is never benign.
|
||||
func TestKernelStep_IsDestructiveClass(t *testing.T) {
|
||||
if reconcile.Classify(reconcile.ClassOSKernelStep, reconcile.Provenance{}) != reconcile.Destructive {
|
||||
t.Fatal("os_kernel_step must be destructive-class (signed, operational key)")
|
||||
}
|
||||
}
|
||||
|
||||
// A kept stage report (the agent was killed mid-stage) reaches the hub as "staged" with its kernel view.
|
||||
func TestKernel_KeptStageReportIsStaged(t *testing.T) {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, tonight(0))
|
||||
ring := 0
|
||||
rep := l.reportFromKept(context.Background(), WrapperReport{Layer: LayerKernel, Mode: "apply", Ring: &ring,
|
||||
Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Kernel: kview("staged")}, "/x/report-r-kernel-apply.json")
|
||||
if rep.Outcome != "staged" || !rep.Healthy || len(rep.Kernel) == 0 {
|
||||
t.Fatalf("kept = %+v", rep)
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,9 @@ const (
|
||||
// LayerPVE is the HOST's Proxmox userspace packages — slow lane (R-812 option A, `09` §3 decision 163, `11` §5.10):
|
||||
// ring 0 in the night leg after a healthy host step, ring 1 only inside a signed os_pve_step. Never a kernel.
|
||||
LayerPVE = "pve"
|
||||
// LayerKernel is the HOST's kernel — the kernel lane (R-836, `09` §3 decision 172, `11` §5.11): a one-shot boot of
|
||||
// the new kernel through the ESP flag, the default moved only after a healthy boot (kernel.go).
|
||||
LayerKernel = "kernel"
|
||||
)
|
||||
|
||||
// PVEOrigin is the origin apt prints for download.proxmox.com (the wrapper's PVE_ORIGIN); InstalledPVEOrigin is how
|
||||
@@ -126,6 +129,11 @@ type WrapperReport struct {
|
||||
OOMCheck json.RawMessage `json:"oom_check"`
|
||||
// PVEManager: the pve layer — pveversion's pve-manager version after the step ("unknown" = unreadable).
|
||||
PVEManager string `json:"pve_manager"`
|
||||
// Kernel (R-836): the kernel layer's view {running, default, flag, phase, from, to, …}; KernelEvent what kernel-boot
|
||||
// found after a boot; OutcomeHint "nothing" when no kernel was pending.
|
||||
Kernel json.RawMessage `json:"kernel"`
|
||||
KernelEvent string `json:"kernel_event"`
|
||||
OutcomeHint string `json:"outcome_hint"`
|
||||
// R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the
|
||||
// agent process that started the pass. ReleaseID / VMID were always in the report.
|
||||
RunID string `json:"run_id"`
|
||||
@@ -168,6 +176,10 @@ type Report struct {
|
||||
OOMCheck json.RawMessage `json:"oom_check,omitempty"`
|
||||
// PVEManager: pve layer — pve-manager's version after the step (the hub's System page; R-812 option A).
|
||||
PVEManager string `json:"pve_manager,omitempty"`
|
||||
// Kernel: kernel layer — the wrapper's kernel view, byte for byte (running, default, flag, phase, from, to). The
|
||||
// outcomes of this layer: staged | applied (the new kernel is the default) | fell_back | health_failed (self-revert
|
||||
// started) | self_reverted | revert_failed | judging | nothing | refused | failed.
|
||||
Kernel json.RawMessage `json:"kernel,omitempty"`
|
||||
|
||||
unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it
|
||||
}
|
||||
@@ -500,7 +512,7 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
|
||||
|
||||
// Pass is one leg's reports; an empty Layer means the step did not run.
|
||||
type Pass struct {
|
||||
Guest, Host, Docker, PVE Report
|
||||
Guest, Host, Docker, PVE, Kernel Report
|
||||
}
|
||||
|
||||
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
|
||||
@@ -542,6 +554,16 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
|
||||
default:
|
||||
p.PVE = l.runPVE(ctx, g.RunID, vmid, trigger, blk, dockerOpts{})
|
||||
}
|
||||
// R-836 (`09` §3 decision 172): the kernel step ENDS the night — an appliance, after a healthy host step (and a
|
||||
// healthy Proxmox step when one ran), only on a night the hub marks as told. It reboots the box. Pinned by TestKernel_*.
|
||||
switch {
|
||||
case !l.Appliance || h.Layer == "" || !okStep(h):
|
||||
lg.Info("osupdate: kernel step skipped — no healthy host step this pass (an appliance only)", "appliance", l.Appliance, "host_outcome", h.Outcome)
|
||||
case p.PVE.Layer != "" && !okStep(p.PVE):
|
||||
lg.Warn("osupdate: kernel step skipped — the Proxmox step did not end healthy", "pve_outcome", p.PVE.Outcome)
|
||||
default:
|
||||
p.Kernel = l.runKernel(ctx, g.RunID, vmid, trigger, blk)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ type fakeWrapper struct {
|
||||
plans []map[string]any
|
||||
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
|
||||
pveGateHeld bool
|
||||
kernelRep map[string][]WrapperReport // R-836: per kernel-layer mode, successive answers (the last one repeats)
|
||||
}
|
||||
|
||||
func yes() *bool { b := true; return &b }
|
||||
@@ -52,9 +53,19 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
|
||||
f.plans = append(f.plans, plan)
|
||||
layer := plan["layer"].(string)
|
||||
ok := guestOK()
|
||||
if layer == LayerHost || layer == LayerPVE {
|
||||
if layer == LayerHost || layer == LayerPVE || layer == LayerKernel {
|
||||
ok = hostOK()
|
||||
}
|
||||
if layer == LayerKernel {
|
||||
if seq := f.kernelRep[plan["mode"].(string)]; len(seq) > 0 {
|
||||
rep := seq[0]
|
||||
if len(seq) > 1 {
|
||||
f.kernelRep[plan["mode"].(string)] = seq[1:]
|
||||
}
|
||||
out, _ := json.Marshal(rep)
|
||||
return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil
|
||||
}
|
||||
}
|
||||
if layer == LayerPVE && plan["mode"] == "apply" {
|
||||
f.pveGateHeld = pvegate.Stepping() // R-812: the /etc/pve write gate must be held while the pve step runs
|
||||
}
|
||||
|
||||
@@ -142,6 +142,17 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
|
||||
default:
|
||||
rep.Outcome = "applied"
|
||||
}
|
||||
if wr.Layer == LayerKernel {
|
||||
// a kernel STAGE changes nothing the box runs (the new kernel only boots once, at the night's reboot), so its
|
||||
// kept copy needs no fresh health reading (R-836)
|
||||
rep.Kernel = rawOrNil(wr.Kernel)
|
||||
rep.Upgraded, rep.PassSeconds, rep.Authority = wr.Upgraded, wr.PassSeconds, wr.Authority
|
||||
if rep.Outcome == "applied" {
|
||||
rep.Outcome = "staged"
|
||||
}
|
||||
rep.Healthy, rep.HealthReason = rep.Outcome == "staged" || rep.Outcome == "nothing", prefix
|
||||
return rep
|
||||
}
|
||||
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
|
||||
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
|
||||
rep.OOMCheck = rawOrNil(wr.OOMCheck)
|
||||
|
||||
@@ -56,6 +56,11 @@ const (
|
||||
// key) like os_docker_step; the root wrapper re-verifies the same signature itself.
|
||||
ClassOSPVEStep OpClass = "os_pve_step"
|
||||
|
||||
// A kernel step on the host (R-836, `09` §3 decision 172, `11` §5.11) — ring 1: it STAGES a kernel (install + the
|
||||
// one-shot flag; the night leg reboots it). Destructive-class (signed, operational key) like os_pve_step; the root
|
||||
// wrapper re-verifies the same signature itself.
|
||||
ClassOSKernelStep OpClass = "os_kernel_step"
|
||||
|
||||
// The config bundle (agent v0.143.0, R-840, `11` §5.4.2): the box's ROOT-OWNED files (sudoers, wrappers, units).
|
||||
// Destructive-class (signed, operational key) like agent_update; the root wrapper re-verifies the signature itself.
|
||||
ClassAgentConfigUpdate OpClass = "agent_config_update"
|
||||
@@ -127,7 +132,7 @@ func Classify(class OpClass, prov Provenance) Disposition {
|
||||
return Destructive
|
||||
case ClassKeyRotation:
|
||||
return Destructive
|
||||
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassAgentConfigUpdate:
|
||||
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassOSKernelStep, ClassAgentConfigUpdate:
|
||||
// Never benign — no agent-internal provenance can make replacing the agent binary
|
||||
// unsigned-safe (a compromised process must not be able to self-bless an update).
|
||||
return Destructive
|
||||
|
||||
Reference in New Issue
Block a user