package osupdate import ( "context" "encoding/base64" "encoding/json" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/hub" "gitea.dooplex.hu/admin/felhom-agent/internal/reconcile" "gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs" ) // ---- the kernel lane (R-836, `09` §3 decision 172, `11` §5.11) ---- const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve" func kview(phase string) json.RawMessage { return mustRaw(KernelView{Running: kOld, Default: kOld, Phase: phase, From: kOld, To: kNew, VMID: 9201}) } func tonight(ring int) *hub.WireOSUpdate { return &hub.WireOSUpdate{Ring: ring, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}} } func kernelCalls(w *fakeWrapper) []string { var m []string for _, p := range w.plans { if p["layer"] == LayerKernel { m = append(m, p["mode"].(string)) } } return m } // Ring 0, a told night: after the healthy host step the leg stages the pending kernel (select pending-kernel, the // kernel the household was told about), tells the hub "staged", THEN reboots — the kernel step ends the night. func TestKernel_Ring0ToldNightStagesThenReboots(t *testing.T) { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{ "kernel-status": {{Kernel: kview("none")}}, "apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}}, "kernel-reboot": {{Kernel: kview("oneshot")}}, }} l, h := newLeg(t, w, tonight(0)) p := l.Run(context.Background(), 9201, "night") if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply,kernel-reboot" { t.Fatalf("kernel calls = %s", got) } var ap map[string]any for _, x := range w.plans { if x["layer"] == LayerKernel && x["mode"] == "apply" { ap = x } } if ap["select"] != "pending-kernel" || ap["expect_kver"] != kNew || ap["lane"] != "slow" { t.Fatalf("stage plan = %v", ap) } if p.Kernel.Outcome != "staged" || !p.Kernel.Healthy { t.Fatalf("kernel report = %+v", p.Kernel) } last := h.reports[len(h.reports)-1] if last.Layer != LayerKernel || last.Outcome != "staged" { t.Fatalf("the hub must hear 'staged' before the reboot: %+v", h.reports) } // the kernel step is the LAST wrapper call of the night if lp := w.plans[len(w.plans)-1]; lp["layer"] != LayerKernel || lp["mode"] != "kernel-reboot" { t.Fatalf("the reboot must end the night, last call = %v", lp) } } // No mail, no step: a kernel the household was NOT told about never runs; nor in a debug pass; nor without a block. // COMPANION RED-PROOF (observed): drop the `!blk.Kernel.Tonight` case in kernelDue → the first sub-case fails. func TestKernel_NoMailNoStep(t *testing.T) { cases := map[string]struct { blk *hub.WireOSUpdate trigger string }{ "not told": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: false}}, "night"}, "debug pass": {tonight(0), "debug"}, "no block": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night"}, "switch off": {&hub.WireOSUpdate{Ring: 0, Enabled: false, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}, "night"}, "bad kver": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: "7.0; reboot", Tonight: true}}, "night"}, } for name, c := range cases { w := &fakeWrapper{t: t} l, _ := newLeg(t, w, c.blk) p := l.Run(context.Background(), 9201, c.trigger) if len(kernelCalls(w)) != 0 || p.Kernel.Layer != "" { t.Fatalf("%s: a kernel step ran: %v", name, kernelCalls(w)) } } } // The kernel step needs a healthy host step on an appliance, and a healthy Proxmox step when one ran. func TestKernel_SkippedWithoutHealthyEarlierSteps(t *testing.T) { w := &fakeWrapper{t: t} l, _ := newLeg(t, w, tonight(0)) l.Appliance = false l.Run(context.Background(), 9201, "night") if len(kernelCalls(w)) != 0 { t.Fatalf("a BYO box took a kernel step: %v", kernelCalls(w)) } w2 := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}}, PVEManager: "9.2.2"}}} // pveversion still old → the pve step is unhealthy l2, _ := newLeg(t, w2, tonight(0)) l2.Run(context.Background(), 9201, "night") if len(kernelCalls(w2)) != 0 { t.Fatalf("a kernel step ran after an unhealthy Proxmox step: %v", kernelCalls(w2)) } } // Ring 1 reboots only a kernel a signed job staged — never stages one itself in the night leg. func TestKernel_Ring1RebootsOnlyASignedStage(t *testing.T) { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}}} l, _ := newLeg(t, w, tonight(1)) l.Run(context.Background(), 9201, "night") if got := strings.Join(kernelCalls(w), ","); got != "kernel-status" { t.Fatalf("ring 1 without a staged kernel: calls = %s", got) } w2 := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("staged")}}, "kernel-reboot": {{Kernel: kview("oneshot")}}}} l2, _ := newLeg(t, w2, tonight(1)) p := l2.Run(context.Background(), 9201, "night") if got := strings.Join(kernelCalls(w2), ","); got != "kernel-status,kernel-reboot" || p.Kernel.Outcome != "staged" { t.Fatalf("ring 1 with a staged kernel: calls = %s report = %+v", got, p.Kernel) } } // A refused stage never reboots. func TestKernel_RefusedStageNeverReboots(t *testing.T) { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}, "apply": {{Refused: json.RawMessage(`{"code":"R20","reason":"/boot/efi is not a mounted vfat ESP"}`)}}}} l, h := newLeg(t, w, tonight(0)) p := l.Run(context.Background(), 9201, "night") if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply" || p.Kernel.Outcome != "refused" { t.Fatalf("calls = %s report = %+v", got, p.Kernel) } if last := h.reports[len(h.reports)-1]; last.Layer != LayerKernel || last.Outcome != "refused" { t.Fatalf("the hub must hear the refusal: %+v", last) } } // THE one-shot boot rule: the host rule AND the hub reached. COMPANION RED-PROOF (observed): drop the hubReached // check in KernelVerdict → the second case fails. func TestKernelVerdict(t *testing.T) { if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, true); !ok { t.Fatalf("a healthy boot read unhealthy: %s", why) } if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, false); ok || !strings.Contains(why, "hub") { t.Fatalf("a box that has not reached the hub must not pass: ok=%v %q", ok, why) } down := hostOK() down.GuestRunning = new(bool) if ok, _ := KernelVerdict(hostOK(), down, hub.TunnelRunning, true); ok { t.Fatal("a guest that does not run must fail") } if ok, _ := KernelVerdict(hostOK(), hostOK(), hub.TunnelUnknown, true); ok { t.Fatal("an unknown tunnel must fail (the host rule)") } } func judgingLeg(t *testing.T, w *fakeWrapper) (*Leg, *fakeHub) { if w.kernelRep == nil { w.kernelRep = map[string][]WrapperReport{} } if _, ok := w.kernelRep["kernel-boot"]; !ok { w.kernelRep["kernel-boot"] = []WrapperReport{{KernelEvent: "judging", Kernel: mustRaw(KernelView{Running: kNew, Default: kOld, Phase: "judging", From: kOld, To: kNew, VMID: 9201}), HealthBefore: hostOK()}} } return newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true}) } // A healthy one-shot boot: the hub hears "judging", then kernel-good, then "applied". func TestKernelAfterBoot_HealthyBecomesTheDefault(t *testing.T) { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-good": {{Kernel: kview("good")}}}} l, h := judgingLeg(t, w) r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: 30 * time.Second}) if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot,health,kernel-good" { t.Fatalf("calls = %s", got) } if r.Outcome != "applied" || !r.Healthy { t.Fatalf("report = %+v", r) } if len(h.reports) != 2 || h.reports[0].Outcome != "judging" || h.reports[1].Outcome != "applied" { t.Fatalf("hub reports = %+v", h.reports) } if w.plans[1]["vmid"] != float64(9201) { t.Fatalf("the health reading must use the step's own guest, got %v", w.plans[1]["vmid"]) } } // An unhealthy one-shot boot: wait the full judge time, tell the hub, then ONE kernel-revert. // COMPANION RED-PROOF (observed): return before the kernel-revert call in judgeKernel → "calls" fails. func TestKernelAfterBoot_UnhealthyRevertsOnceAfterTheWait(t *testing.T) { down := hostOK() down.GuestRunning = new(bool) w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"health": {{Health: down}}, "kernel-revert": {{Kernel: kview("reverting")}}}} l, h := judgingLeg(t, w) start := l.now() r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: time.Minute}) calls := kernelCalls(w) if calls[len(calls)-1] != "kernel-revert" || strings.Count(strings.Join(calls, ","), "kernel-revert") != 1 { t.Fatalf("calls = %v", calls) } if waited := l.now().Sub(start); waited < 10*time.Minute { t.Fatalf("reverted after %s — before the judge wait", waited) } if r.Outcome != "health_failed" || !strings.Contains(r.HealthReason, "not running") { t.Fatalf("report = %+v", r) } if last := h.reports[len(h.reports)-1]; last.Outcome != "health_failed" { t.Fatalf("the hub must hear health_failed before the revert reboot: %+v", h.reports) } for _, p := range w.plans { if p["mode"] == "kernel-revert" && !strings.Contains(p["reason"].(string), "not running") { t.Fatalf("the revert must carry the reason: %v", p) } } } // A box that never reaches the hub is not "healthy" — it reverts too. func TestKernelAfterBoot_NoHubMeansRevert(t *testing.T) { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-revert": {{Kernel: kview("reverting")}}}} l, _ := judgingLeg(t, w) l.Hub = unreachableHub{} l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 5 * time.Minute, Poll: time.Minute}) if c := kernelCalls(w); c[len(c)-1] != "kernel-revert" { t.Fatalf("calls = %v", c) } } type unreachableHub struct{} func (unreachableHub) PostOSReport(context.Context, []byte) error { return context.DeadlineExceeded } // What kernel-boot found becomes the hub's outcome, with no judging and no reboot. func TestKernelAfterBoot_FallBackAndRevertResultsAreReported(t *testing.T) { for ev, want := range map[string]string{"fell_back": "fell_back", "self_reverted": "self_reverted", "revert_failed": "revert_failed"} { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: ev, Kernel: mustRaw(KernelView{Running: kOld, Default: kOld, Phase: ev, From: kOld, To: kNew, Reason: "r", VMID: 9201})}}}} l, h := judgingLeg(t, w) r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{}) if r.Outcome != want || len(h.reports) != 1 || h.reports[0].Outcome != want { t.Fatalf("%s: report %+v hub %+v", ev, r, h.reports) } if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot" { t.Fatalf("%s: calls = %s", ev, got) } } // nothing to do → nothing reported w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: "none", Kernel: kview("good")}}}} l, h := judgingLeg(t, w) if r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{}); r.Layer != "" || len(h.reports) != 0 { t.Fatalf("an ordinary boot must report nothing: %+v %+v", r, h.reports) } } // The signed executor STAGES (listed + the raw envelope + the kver) and never reboots. func TestKernelStepExecutor_StagesNeverReboots(t *testing.T) { w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "signed", Kernel: kview("staged")}}}} l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true}) e := KernelStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }} params, _ := json.Marshal(KernelStepParams{ReleaseID: "os-kernel-1", Kver: kNew, Packages: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22", Origin: PVEOrigin}}}) ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_kernel_step"}`), Sig: []byte("SIG")}) if err := e.Execute(ctx, OpKernelStep, params); err != nil { t.Fatal(err) } pp := w.plans[len(w.plans)-1] sg, _ := pp["signed"].(map[string]any) if pp["mode"] != "apply" || pp["select"] != "listed" || pp["expect_kver"] != kNew || sg == nil || sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_kernel_step"}`)) { t.Fatalf("plan = %v", pp) } if got := strings.Join(kernelCalls(w), ","); got != "apply" { t.Fatalf("a signed stage must never reboot: %s", got) } if len(h.reports) != 1 || h.reports[0].Outcome != "staged" { t.Fatalf("hub = %+v", h.reports) } if err := e.Execute(context.Background(), OpKernelStep, params); err == nil { t.Fatal("no envelope must refuse") } bad, _ := json.Marshal(KernelStepParams{Kver: "x", Packages: []Package{{Name: "a"}}}) if err := e.Execute(ctx, OpKernelStep, bad); err == nil { t.Fatal("a bad kver must refuse") } if err := e.Execute(ctx, OpPVEStep, params); err != signedjobs.ErrNoExecutor { t.Fatalf("another op must pass through the chain: %v", err) } } // os_kernel_step is never benign. func TestKernelStep_IsDestructiveClass(t *testing.T) { if reconcile.Classify(reconcile.ClassOSKernelStep, reconcile.Provenance{}) != reconcile.Destructive { t.Fatal("os_kernel_step must be destructive-class (signed, operational key)") } } // A kept stage report (the agent was killed mid-stage) reaches the hub as "staged" with its kernel view. func TestKernel_KeptStageReportIsStaged(t *testing.T) { w := &fakeWrapper{t: t} l, _ := newLeg(t, w, tonight(0)) ring := 0 rep := l.reportFromKept(context.Background(), WrapperReport{Layer: LayerKernel, Mode: "apply", Ring: &ring, Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Kernel: kview("staged")}, "/x/report-r-kernel-apply.json") if rep.Outcome != "staged" || !rep.Healthy || len(rep.Kernel) == 0 { t.Fatalf("kept = %+v", rep) } }