felhom-agent/internal/osupdate/kernel_test.go

package osupdate

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
	"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
	"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)

// ---- the kernel lane (R-836, `09` §3 decision 172, `11` §5.11) ----

const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve"

func kview(phase string) json.RawMessage {
	return mustRaw(KernelView{Running: kOld, Default: kOld, Phase: phase, From: kOld, To: kNew, VMID: 9201})
}

func tonight(ring int) *hub.WireOSUpdate {
	return &hub.WireOSUpdate{Ring: ring, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}
}

func kernelCalls(w *fakeWrapper) []string {
	var m []string
	for _, p := range w.plans {
		if p["layer"] == LayerKernel {
			m = append(m, p["mode"].(string))
		}
	}
	return m
}

// Ring 0, a told night: after the healthy host step the leg stages the pending kernel (select pending-kernel, the
// kernel the household was told about), tells the hub "staged", THEN reboots — the kernel step ends the night.
func TestKernel_Ring0ToldNightStagesThenReboots(t *testing.T) {
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{
		"kernel-status": {{Kernel: kview("none")}},
		"apply":         {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}},
		"kernel-reboot": {{Kernel: kview("oneshot")}},
	}}
	l, h := newLeg(t, w, tonight(0))
	p := l.Run(context.Background(), 9201, "night")
	if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply,kernel-reboot" {
		t.Fatalf("kernel calls = %s", got)
	}
	var ap map[string]any
	for _, x := range w.plans {
		if x["layer"] == LayerKernel && x["mode"] == "apply" {
			ap = x
		}
	}
	if ap["select"] != "pending-kernel" || ap["expect_kver"] != kNew || ap["lane"] != "slow" {
		t.Fatalf("stage plan = %v", ap)
	}
	if p.Kernel.Outcome != "staged" || !p.Kernel.Healthy {
		t.Fatalf("kernel report = %+v", p.Kernel)
	}
	last := h.reports[len(h.reports)-1]
	if last.Layer != LayerKernel || last.Outcome != "staged" {
		t.Fatalf("the hub must hear 'staged' before the reboot: %+v", h.reports)
	}
	// the kernel step is the LAST wrapper call of the night
	if lp := w.plans[len(w.plans)-1]; lp["layer"] != LayerKernel || lp["mode"] != "kernel-reboot" {
		t.Fatalf("the reboot must end the night, last call = %v", lp)
	}
}

// No mail, no step: a kernel the household was NOT told about never runs; nor in a debug pass; nor without a block.
// COMPANION RED-PROOF (observed): drop the `!blk.Kernel.Tonight` case in kernelDue → the first sub-case fails.
func TestKernel_NoMailNoStep(t *testing.T) {
	cases := map[string]struct {
		blk     *hub.WireOSUpdate
		trigger string
	}{
		"not told":   {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: false}}, "night"},
		"debug pass": {tonight(0), "debug"},
		"no block":   {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night"},
		"switch off": {&hub.WireOSUpdate{Ring: 0, Enabled: false, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}, "night"},
		"bad kver":   {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: "7.0; reboot", Tonight: true}}, "night"},
	}
	for name, c := range cases {
		w := &fakeWrapper{t: t}
		l, _ := newLeg(t, w, c.blk)
		p := l.Run(context.Background(), 9201, c.trigger)
		if len(kernelCalls(w)) != 0 || p.Kernel.Layer != "" {
			t.Fatalf("%s: a kernel step ran: %v", name, kernelCalls(w))
		}
	}
}

// The kernel step needs a healthy host step on an appliance, and a healthy Proxmox step when one ran.
func TestKernel_SkippedWithoutHealthyEarlierSteps(t *testing.T) {
	w := &fakeWrapper{t: t}
	l, _ := newLeg(t, w, tonight(0))
	l.Appliance = false
	l.Run(context.Background(), 9201, "night")
	if len(kernelCalls(w)) != 0 {
		t.Fatalf("a BYO box took a kernel step: %v", kernelCalls(w))
	}
	w2 := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}},
		PVEManager: "9.2.2"}}} // pveversion still old → the pve step is unhealthy
	l2, _ := newLeg(t, w2, tonight(0))
	l2.Run(context.Background(), 9201, "night")
	if len(kernelCalls(w2)) != 0 {
		t.Fatalf("a kernel step ran after an unhealthy Proxmox step: %v", kernelCalls(w2))
	}
}

// Ring 1 reboots only a kernel a signed job staged — never stages one itself in the night leg.
func TestKernel_Ring1RebootsOnlyASignedStage(t *testing.T) {
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}}}
	l, _ := newLeg(t, w, tonight(1))
	l.Run(context.Background(), 9201, "night")
	if got := strings.Join(kernelCalls(w), ","); got != "kernel-status" {
		t.Fatalf("ring 1 without a staged kernel: calls = %s", got)
	}
	w2 := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("staged")}},
		"kernel-reboot": {{Kernel: kview("oneshot")}}}}
	l2, _ := newLeg(t, w2, tonight(1))
	p := l2.Run(context.Background(), 9201, "night")
	if got := strings.Join(kernelCalls(w2), ","); got != "kernel-status,kernel-reboot" || p.Kernel.Outcome != "staged" {
		t.Fatalf("ring 1 with a staged kernel: calls = %s report = %+v", got, p.Kernel)
	}
}

// A refused stage never reboots.
func TestKernel_RefusedStageNeverReboots(t *testing.T) {
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}},
		"apply": {{Refused: json.RawMessage(`{"code":"R20","reason":"/boot/efi is not a mounted vfat ESP"}`)}}}}
	l, h := newLeg(t, w, tonight(0))
	p := l.Run(context.Background(), 9201, "night")
	if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply" || p.Kernel.Outcome != "refused" {
		t.Fatalf("calls = %s report = %+v", got, p.Kernel)
	}
	if last := h.reports[len(h.reports)-1]; last.Layer != LayerKernel || last.Outcome != "refused" {
		t.Fatalf("the hub must hear the refusal: %+v", last)
	}
}

// THE one-shot boot rule: the host rule AND the hub reached. COMPANION RED-PROOF (observed): drop the hubReached
// check in KernelVerdict → the second case fails.
func TestKernelVerdict(t *testing.T) {
	if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, true); !ok {
		t.Fatalf("a healthy boot read unhealthy: %s", why)
	}
	if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, false); ok || !strings.Contains(why, "hub") {
		t.Fatalf("a box that has not reached the hub must not pass: ok=%v %q", ok, why)
	}
	down := hostOK()
	down.GuestRunning = new(bool)
	if ok, _ := KernelVerdict(hostOK(), down, hub.TunnelRunning, true); ok {
		t.Fatal("a guest that does not run must fail")
	}
	if ok, _ := KernelVerdict(hostOK(), hostOK(), hub.TunnelUnknown, true); ok {
		t.Fatal("an unknown tunnel must fail (the host rule)")
	}
}

func judgingLeg(t *testing.T, w *fakeWrapper) (*Leg, *fakeHub) {
	if w.kernelRep == nil {
		w.kernelRep = map[string][]WrapperReport{}
	}
	if _, ok := w.kernelRep["kernel-boot"]; !ok {
		w.kernelRep["kernel-boot"] = []WrapperReport{{KernelEvent: "judging", Kernel: mustRaw(KernelView{Running: kNew,
			Default: kOld, Phase: "judging", From: kOld, To: kNew, VMID: 9201}), HealthBefore: hostOK()}}
	}
	return newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
}

// A healthy one-shot boot: the hub hears "judging", then kernel-good, then "applied".
func TestKernelAfterBoot_HealthyBecomesTheDefault(t *testing.T) {
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-good": {{Kernel: kview("good")}}}}
	l, h := judgingLeg(t, w)
	r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: 30 * time.Second})
	if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot,health,kernel-good" {
		t.Fatalf("calls = %s", got)
	}
	if r.Outcome != "applied" || !r.Healthy {
		t.Fatalf("report = %+v", r)
	}
	if len(h.reports) != 2 || h.reports[0].Outcome != "judging" || h.reports[1].Outcome != "applied" {
		t.Fatalf("hub reports = %+v", h.reports)
	}
	if w.plans[1]["vmid"] != float64(9201) {
		t.Fatalf("the health reading must use the step's own guest, got %v", w.plans[1]["vmid"])
	}
}

// An unhealthy one-shot boot: wait the full judge time, tell the hub, then ONE kernel-revert.
// COMPANION RED-PROOF (observed): return before the kernel-revert call in judgeKernel → "calls" fails.
func TestKernelAfterBoot_UnhealthyRevertsOnceAfterTheWait(t *testing.T) {
	down := hostOK()
	down.GuestRunning = new(bool)
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"health": {{Health: down}},
		"kernel-revert": {{Kernel: kview("reverting")}}}}
	l, h := judgingLeg(t, w)
	start := l.now()
	r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: time.Minute})
	calls := kernelCalls(w)
	if calls[len(calls)-1] != "kernel-revert" || strings.Count(strings.Join(calls, ","), "kernel-revert") != 1 {
		t.Fatalf("calls = %v", calls)
	}
	if waited := l.now().Sub(start); waited < 10*time.Minute {
		t.Fatalf("reverted after %s — before the judge wait", waited)
	}
	if r.Outcome != "health_failed" || !strings.Contains(r.HealthReason, "not running") {
		t.Fatalf("report = %+v", r)
	}
	if last := h.reports[len(h.reports)-1]; last.Outcome != "health_failed" {
		t.Fatalf("the hub must hear health_failed before the revert reboot: %+v", h.reports)
	}
	for _, p := range w.plans {
		if p["mode"] == "kernel-revert" && !strings.Contains(p["reason"].(string), "not running") {
			t.Fatalf("the revert must carry the reason: %v", p)
		}
	}
}

// A box that never reaches the hub is not "healthy" — it reverts too.
func TestKernelAfterBoot_NoHubMeansRevert(t *testing.T) {
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-revert": {{Kernel: kview("reverting")}}}}
	l, _ := judgingLeg(t, w)
	l.Hub = unreachableHub{}
	l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 5 * time.Minute, Poll: time.Minute})
	if c := kernelCalls(w); c[len(c)-1] != "kernel-revert" {
		t.Fatalf("calls = %v", c)
	}
}

type unreachableHub struct{}

func (unreachableHub) PostOSReport(context.Context, []byte) error { return context.DeadlineExceeded }

// What kernel-boot found becomes the hub's outcome, with no judging and no reboot.
func TestKernelAfterBoot_FallBackAndRevertResultsAreReported(t *testing.T) {
	for ev, want := range map[string]string{"fell_back": "fell_back", "self_reverted": "self_reverted", "revert_failed": "revert_failed"} {
		w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: ev,
			Kernel: mustRaw(KernelView{Running: kOld, Default: kOld, Phase: ev, From: kOld, To: kNew, Reason: "r", VMID: 9201})}}}}
		l, h := judgingLeg(t, w)
		r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{})
		if r.Outcome != want || len(h.reports) != 1 || h.reports[0].Outcome != want {
			t.Fatalf("%s: report %+v hub %+v", ev, r, h.reports)
		}
		if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot" {
			t.Fatalf("%s: calls = %s", ev, got)
		}
	}
	// nothing to do → nothing reported
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: "none", Kernel: kview("good")}}}}
	l, h := judgingLeg(t, w)
	if r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{}); r.Layer != "" || len(h.reports) != 0 {
		t.Fatalf("an ordinary boot must report nothing: %+v %+v", r, h.reports)
	}
}

// The signed executor STAGES (listed + the raw envelope + the kver) and never reboots.
func TestKernelStepExecutor_StagesNeverReboots(t *testing.T) {
	w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0",
		Version: "7.0.14-22"}}, Authority: "signed", Kernel: kview("staged")}}}}
	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
	e := KernelStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
	params, _ := json.Marshal(KernelStepParams{ReleaseID: "os-kernel-1", Kver: kNew,
		Packages: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22", Origin: PVEOrigin}}})
	ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_kernel_step"}`), Sig: []byte("SIG")})
	if err := e.Execute(ctx, OpKernelStep, params); err != nil {
		t.Fatal(err)
	}
	pp := w.plans[len(w.plans)-1]
	sg, _ := pp["signed"].(map[string]any)
	if pp["mode"] != "apply" || pp["select"] != "listed" || pp["expect_kver"] != kNew || sg == nil ||
		sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_kernel_step"}`)) {
		t.Fatalf("plan = %v", pp)
	}
	if got := strings.Join(kernelCalls(w), ","); got != "apply" {
		t.Fatalf("a signed stage must never reboot: %s", got)
	}
	if len(h.reports) != 1 || h.reports[0].Outcome != "staged" {
		t.Fatalf("hub = %+v", h.reports)
	}
	if err := e.Execute(context.Background(), OpKernelStep, params); err == nil {
		t.Fatal("no envelope must refuse")
	}
	bad, _ := json.Marshal(KernelStepParams{Kver: "x", Packages: []Package{{Name: "a"}}})
	if err := e.Execute(ctx, OpKernelStep, bad); err == nil {
		t.Fatal("a bad kver must refuse")
	}
	if err := e.Execute(ctx, OpPVEStep, params); err != signedjobs.ErrNoExecutor {
		t.Fatalf("another op must pass through the chain: %v", err)
	}
}

// os_kernel_step is never benign.
func TestKernelStep_IsDestructiveClass(t *testing.T) {
	if reconcile.Classify(reconcile.ClassOSKernelStep, reconcile.Provenance{}) != reconcile.Destructive {
		t.Fatal("os_kernel_step must be destructive-class (signed, operational key)")
	}
}

// A kept stage report (the agent was killed mid-stage) reaches the hub as "staged" with its kernel view.
func TestKernel_KeptStageReportIsStaged(t *testing.T) {
	w := &fakeWrapper{t: t}
	l, _ := newLeg(t, w, tonight(0))
	ring := 0
	rep := l.reportFromKept(context.Background(), WrapperReport{Layer: LayerKernel, Mode: "apply", Ring: &ring,
		Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Kernel: kview("staged")}, "/x/report-r-kernel-apply.json")
	if rep.Outcome != "staged" || !rep.Healthy || len(rep.Kernel) == 0 {
		t.Fatalf("kept = %+v", rep)
	}
}