513 lines
23 KiB
Go
513 lines
23 KiB
Go
package osupdate
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
|
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
|
)
|
|
|
|
// ---- the kernel lane (R-836, `09` §3 decision 172, `11` §5.11) ----
|
|
|
|
const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve"
|
|
|
|
func kview(phase string) json.RawMessage {
|
|
return mustRaw(KernelView{Running: kOld, Default: kOld, Phase: phase, From: kOld, To: kNew, VMID: 9201})
|
|
}
|
|
|
|
func tonight(ring int) *hub.WireOSUpdate {
|
|
return &hub.WireOSUpdate{Ring: ring, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}
|
|
}
|
|
|
|
func kernelCalls(w *fakeWrapper) []string {
|
|
var m []string
|
|
for _, p := range w.plans {
|
|
if p["layer"] == LayerKernel && p["mode"] != "kernel-cleanup" { // the kernel STEP's calls (the clean-up: cleanupCalls)
|
|
m = append(m, p["mode"].(string))
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
func cleanupCalls(w *fakeWrapper) []map[string]any {
|
|
var m []map[string]any
|
|
for _, p := range w.plans {
|
|
if p["layer"] == LayerKernel && p["mode"] == "kernel-cleanup" {
|
|
m = append(m, p)
|
|
}
|
|
}
|
|
return m
|
|
}
|
|
|
|
// Ring 0, a told night: after the healthy host step the leg stages the pending kernel (select pending-kernel, the
|
|
// kernel the household was told about), tells the hub "staged", THEN reboots — the kernel step ends the night.
|
|
func TestKernel_Ring0ToldNightStagesThenReboots(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{
|
|
"kernel-status": {{Kernel: kview("none")}},
|
|
"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}},
|
|
"kernel-reboot": {{Kernel: kview("oneshot")}},
|
|
}}
|
|
l, h := newLeg(t, w, tonight(0))
|
|
p := l.Run(context.Background(), 9201, "night")
|
|
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply,kernel-reboot" {
|
|
t.Fatalf("kernel calls = %s", got)
|
|
}
|
|
var ap map[string]any
|
|
for _, x := range w.plans {
|
|
if x["layer"] == LayerKernel && x["mode"] == "apply" {
|
|
ap = x
|
|
}
|
|
}
|
|
// R-898: EXACTLY the told kernel — the listed set derived from it, never "pending" (red before the fix: select was
|
|
// pending-kernel, so a newer kernel in the sources by night was refused R23 and the night was lost)
|
|
pk, _ := json.Marshal(ap["packages"])
|
|
if ap["select"] != "listed" || ap["expect_kver"] != kNew || ap["lane"] != "slow" ||
|
|
string(pk) != `[{"name":"proxmox-kernel-7.0","origin":"Proxmox Debian Repository","version":"7.0.14-22"},{"name":"proxmox-kernel-7.0.14-22-pve-signed","origin":"Proxmox Debian Repository","version":"7.0.14-22"}]` {
|
|
t.Fatalf("stage plan = %v (packages %s)", ap, pk)
|
|
}
|
|
if p.Kernel.Outcome != "staged" || !p.Kernel.Healthy {
|
|
t.Fatalf("kernel report = %+v", p.Kernel)
|
|
}
|
|
last := h.reports[len(h.reports)-1]
|
|
if last.Layer != LayerKernel || last.Outcome != "staged" {
|
|
t.Fatalf("the hub must hear 'staged' before the reboot: %+v", h.reports)
|
|
}
|
|
// the kernel step is the LAST wrapper call of the night
|
|
if lp := w.plans[len(w.plans)-1]; lp["layer"] != LayerKernel || lp["mode"] != "kernel-reboot" {
|
|
t.Fatalf("the reboot must end the night, last call = %v", lp)
|
|
}
|
|
}
|
|
|
|
// No mail, no step: a kernel the household was NOT told about never runs; nor in a debug pass; nor without a block.
|
|
// COMPANION RED-PROOF (observed): drop the `!blk.Kernel.Tonight` case in kernelDue → the first sub-case fails.
|
|
func TestKernel_NoMailNoStep(t *testing.T) {
|
|
cases := map[string]struct {
|
|
blk *hub.WireOSUpdate
|
|
trigger string
|
|
}{
|
|
"not told": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: false}}, "night"},
|
|
"debug pass": {tonight(0), "debug"},
|
|
"no block": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night"},
|
|
"switch off": {&hub.WireOSUpdate{Ring: 0, Enabled: false, Kernel: &hub.WireKernelStep{Kver: kNew, Tonight: true}}, "night"},
|
|
"bad kver": {&hub.WireOSUpdate{Ring: 0, Enabled: true, Kernel: &hub.WireKernelStep{Kver: "7.0; reboot", Tonight: true}}, "night"},
|
|
}
|
|
for name, c := range cases {
|
|
w := &fakeWrapper{t: t}
|
|
l, _ := newLeg(t, w, c.blk)
|
|
p := l.Run(context.Background(), 9201, c.trigger)
|
|
if len(kernelCalls(w)) != 0 || p.Kernel.Layer != "" {
|
|
t.Fatalf("%s: a kernel step ran: %v", name, kernelCalls(w))
|
|
}
|
|
}
|
|
}
|
|
|
|
// The kernel step needs a healthy host step on an appliance, and a healthy Proxmox step when one ran.
|
|
func TestKernel_SkippedWithoutHealthyEarlierSteps(t *testing.T) {
|
|
w := &fakeWrapper{t: t}
|
|
l, _ := newLeg(t, w, tonight(0))
|
|
l.Appliance = false
|
|
l.Run(context.Background(), 9201, "night")
|
|
if len(kernelCalls(w)) != 0 {
|
|
t.Fatalf("a BYO box took a kernel step: %v", kernelCalls(w))
|
|
}
|
|
w2 := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}},
|
|
PVEManager: "9.2.2"}}} // pveversion still old → the pve step is unhealthy
|
|
l2, _ := newLeg(t, w2, tonight(0))
|
|
l2.Run(context.Background(), 9201, "night")
|
|
if len(kernelCalls(w2)) != 0 {
|
|
t.Fatalf("a kernel step ran after an unhealthy Proxmox step: %v", kernelCalls(w2))
|
|
}
|
|
}
|
|
|
|
// Ring 1 reboots only a kernel a signed job staged — never stages one itself in the night leg.
|
|
func TestKernel_Ring1RebootsOnlyASignedStage(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}}}
|
|
l, _ := newLeg(t, w, tonight(1))
|
|
l.Run(context.Background(), 9201, "night")
|
|
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status" {
|
|
t.Fatalf("ring 1 without a staged kernel: calls = %s", got)
|
|
}
|
|
w2 := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("staged")}},
|
|
"kernel-reboot": {{Kernel: kview("oneshot")}}}}
|
|
l2, _ := newLeg(t, w2, tonight(1))
|
|
p := l2.Run(context.Background(), 9201, "night")
|
|
if got := strings.Join(kernelCalls(w2), ","); got != "kernel-status,kernel-reboot" || p.Kernel.Outcome != "staged" {
|
|
t.Fatalf("ring 1 with a staged kernel: calls = %s report = %+v", got, p.Kernel)
|
|
}
|
|
}
|
|
|
|
// A refused stage never reboots.
|
|
func TestKernel_RefusedStageNeverReboots(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}},
|
|
"apply": {{Refused: json.RawMessage(`{"code":"R20","reason":"/boot/efi is not a mounted vfat ESP"}`)}}}}
|
|
l, h := newLeg(t, w, tonight(0))
|
|
p := l.Run(context.Background(), 9201, "night")
|
|
if got := strings.Join(kernelCalls(w), ","); got != "kernel-status,apply" || p.Kernel.Outcome != "refused" {
|
|
t.Fatalf("calls = %s report = %+v", got, p.Kernel)
|
|
}
|
|
if last := h.reports[len(h.reports)-1]; last.Layer != LayerKernel || last.Outcome != "refused" {
|
|
t.Fatalf("the hub must hear the refusal: %+v", last)
|
|
}
|
|
}
|
|
|
|
// THE one-shot boot rule: the host rule AND the hub reached. COMPANION RED-PROOF (observed): drop the hubReached
|
|
// check in KernelVerdict → the second case fails.
|
|
func TestKernelVerdict(t *testing.T) {
|
|
if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, true); !ok {
|
|
t.Fatalf("a healthy boot read unhealthy: %s", why)
|
|
}
|
|
if ok, why := KernelVerdict(hostOK(), hostOK(), hub.TunnelRunning, false); ok || !strings.Contains(why, "hub") {
|
|
t.Fatalf("a box that has not reached the hub must not pass: ok=%v %q", ok, why)
|
|
}
|
|
down := hostOK()
|
|
down.GuestRunning = new(bool)
|
|
if ok, _ := KernelVerdict(hostOK(), down, hub.TunnelRunning, true); ok {
|
|
t.Fatal("a guest that does not run must fail")
|
|
}
|
|
if ok, _ := KernelVerdict(hostOK(), hostOK(), hub.TunnelUnknown, true); ok {
|
|
t.Fatal("an unknown tunnel must fail (the host rule)")
|
|
}
|
|
}
|
|
|
|
func judgingLeg(t *testing.T, w *fakeWrapper) (*Leg, *fakeHub) {
|
|
if w.kernelRep == nil {
|
|
w.kernelRep = map[string][]WrapperReport{}
|
|
}
|
|
if _, ok := w.kernelRep["kernel-boot"]; !ok {
|
|
w.kernelRep["kernel-boot"] = []WrapperReport{{KernelEvent: "judging", Kernel: mustRaw(KernelView{Running: kNew,
|
|
Default: kOld, Phase: "judging", From: kOld, To: kNew, VMID: 9201}), HealthBefore: hostOK()}}
|
|
}
|
|
return newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
|
}
|
|
|
|
// A healthy one-shot boot: the hub hears "judging", then kernel-good, then "applied".
|
|
func TestKernelAfterBoot_HealthyBecomesTheDefault(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-good": {{Kernel: kview("good")}}}}
|
|
l, h := judgingLeg(t, w)
|
|
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: 30 * time.Second})
|
|
if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot,health,kernel-good" {
|
|
t.Fatalf("calls = %s", got)
|
|
}
|
|
if r.Outcome != "applied" || !r.Healthy {
|
|
t.Fatalf("report = %+v", r)
|
|
}
|
|
if len(h.reports) != 2 || h.reports[0].Outcome != "judging" || h.reports[1].Outcome != "applied" {
|
|
t.Fatalf("hub reports = %+v", h.reports)
|
|
}
|
|
if w.plans[1]["vmid"] != float64(9201) {
|
|
t.Fatalf("the health reading must use the step's own guest, got %v", w.plans[1]["vmid"])
|
|
}
|
|
}
|
|
|
|
// An unhealthy one-shot boot: wait the full judge time, tell the hub, then ONE kernel-revert.
|
|
// COMPANION RED-PROOF (observed): return before the kernel-revert call in judgeKernel → "calls" fails.
|
|
func TestKernelAfterBoot_UnhealthyRevertsOnceAfterTheWait(t *testing.T) {
|
|
down := hostOK()
|
|
down.GuestRunning = new(bool)
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"health": {{Health: down}},
|
|
"kernel-revert": {{Kernel: kview("reverting")}}}}
|
|
l, h := judgingLeg(t, w)
|
|
start := l.now()
|
|
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 10 * time.Minute, Poll: time.Minute})
|
|
calls := kernelCalls(w)
|
|
if calls[len(calls)-1] != "kernel-revert" || strings.Count(strings.Join(calls, ","), "kernel-revert") != 1 {
|
|
t.Fatalf("calls = %v", calls)
|
|
}
|
|
if waited := l.now().Sub(start); waited < 10*time.Minute {
|
|
t.Fatalf("reverted after %s — before the judge wait", waited)
|
|
}
|
|
if r.Outcome != "health_failed" || !strings.Contains(r.HealthReason, "not running") {
|
|
t.Fatalf("report = %+v", r)
|
|
}
|
|
if last := h.reports[len(h.reports)-1]; last.Outcome != "health_failed" {
|
|
t.Fatalf("the hub must hear health_failed before the revert reboot: %+v", h.reports)
|
|
}
|
|
for _, p := range w.plans {
|
|
if p["mode"] == "kernel-revert" && !strings.Contains(p["reason"].(string), "not running") {
|
|
t.Fatalf("the revert must carry the reason: %v", p)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A box that never reaches the hub is not "healthy" — it reverts too.
|
|
func TestKernelAfterBoot_NoHubMeansRevert(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-revert": {{Kernel: kview("reverting")}}}}
|
|
l, _ := judgingLeg(t, w)
|
|
l.Hub = unreachableHub{}
|
|
l.KernelAfterBoot(context.Background(), 0, KernelJudge{Wait: 5 * time.Minute, Poll: time.Minute})
|
|
if c := kernelCalls(w); c[len(c)-1] != "kernel-revert" {
|
|
t.Fatalf("calls = %v", c)
|
|
}
|
|
}
|
|
|
|
type unreachableHub struct{}
|
|
|
|
func (unreachableHub) PostOSReport(context.Context, []byte) error { return context.DeadlineExceeded }
|
|
|
|
// What kernel-boot found becomes the hub's outcome, with no judging and no reboot.
|
|
func TestKernelAfterBoot_FallBackAndRevertResultsAreReported(t *testing.T) {
|
|
for ev, want := range map[string]string{"fell_back": "fell_back", "self_reverted": "self_reverted", "revert_failed": "revert_failed"} {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: ev,
|
|
Kernel: mustRaw(KernelView{Running: kOld, Default: kOld, Phase: ev, From: kOld, To: kNew, Reason: "r", VMID: 9201})}}}}
|
|
l, h := judgingLeg(t, w)
|
|
r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{})
|
|
if r.Outcome != want || len(h.reports) != 1 || h.reports[0].Outcome != want {
|
|
t.Fatalf("%s: report %+v hub %+v", ev, r, h.reports)
|
|
}
|
|
if got := strings.Join(kernelCalls(w), ","); got != "kernel-boot" {
|
|
t.Fatalf("%s: calls = %s", ev, got)
|
|
}
|
|
}
|
|
// nothing to do → nothing reported
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-boot": {{KernelEvent: "none", Kernel: kview("good")}}}}
|
|
l, h := judgingLeg(t, w)
|
|
if r := l.KernelAfterBoot(context.Background(), 0, KernelJudge{}); r.Layer != "" || len(h.reports) != 0 {
|
|
t.Fatalf("an ordinary boot must report nothing: %+v %+v", r, h.reports)
|
|
}
|
|
}
|
|
|
|
// The signed executor STAGES (listed + the raw envelope + the kver) and never reboots.
|
|
func TestKernelStepExecutor_StagesNeverReboots(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0",
|
|
Version: "7.0.14-22"}}, Authority: "signed", Kernel: kview("staged")}}}}
|
|
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
|
e := KernelStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
|
|
params, _ := json.Marshal(KernelStepParams{ReleaseID: "os-kernel-1", Kver: kNew,
|
|
Packages: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22", Origin: PVEOrigin}}})
|
|
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_kernel_step"}`), Sig: []byte("SIG")})
|
|
if err := e.Execute(ctx, OpKernelStep, params); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pp := w.plans[len(w.plans)-1]
|
|
sg, _ := pp["signed"].(map[string]any)
|
|
if pp["mode"] != "apply" || pp["select"] != "listed" || pp["expect_kver"] != kNew || sg == nil ||
|
|
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_kernel_step"}`)) {
|
|
t.Fatalf("plan = %v", pp)
|
|
}
|
|
if got := strings.Join(kernelCalls(w), ","); got != "apply" {
|
|
t.Fatalf("a signed stage must never reboot: %s", got)
|
|
}
|
|
if len(h.reports) != 1 || h.reports[0].Outcome != "staged" {
|
|
t.Fatalf("hub = %+v", h.reports)
|
|
}
|
|
if err := e.Execute(context.Background(), OpKernelStep, params); err == nil {
|
|
t.Fatal("no envelope must refuse")
|
|
}
|
|
bad, _ := json.Marshal(KernelStepParams{Kver: "x", Packages: []Package{{Name: "a"}}})
|
|
if err := e.Execute(ctx, OpKernelStep, bad); err == nil {
|
|
t.Fatal("a bad kver must refuse")
|
|
}
|
|
if err := e.Execute(ctx, OpPVEStep, params); err != signedjobs.ErrNoExecutor {
|
|
t.Fatalf("another op must pass through the chain: %v", err)
|
|
}
|
|
}
|
|
|
|
// os_kernel_step is never benign.
|
|
func TestKernelStep_IsDestructiveClass(t *testing.T) {
|
|
if reconcile.Classify(reconcile.ClassOSKernelStep, reconcile.Provenance{}) != reconcile.Destructive {
|
|
t.Fatal("os_kernel_step must be destructive-class (signed, operational key)")
|
|
}
|
|
}
|
|
|
|
// A kept stage report (the agent was killed mid-stage) reaches the hub as "staged" with its kernel view.
|
|
func TestKernel_KeptStageReportIsStaged(t *testing.T) {
|
|
w := &fakeWrapper{t: t}
|
|
l, _ := newLeg(t, w, tonight(0))
|
|
ring := 0
|
|
rep := l.reportFromKept(context.Background(), WrapperReport{Layer: LayerKernel, Mode: "apply", Ring: &ring,
|
|
Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Kernel: kview("staged")}, "/x/report-r-kernel-apply.json")
|
|
if rep.Outcome != "staged" || !rep.Healthy || len(rep.Kernel) == 0 {
|
|
t.Fatalf("kept = %+v", rep)
|
|
}
|
|
}
|
|
|
|
func TestKernelSet(t *testing.T) {
|
|
if got := KernelSet("7.0.14-20-pve"); len(got) != 2 || got[0].Name != "proxmox-kernel-7.0" || got[0].Version != "7.0.14-20" ||
|
|
got[1].Name != "proxmox-kernel-7.0.14-20-pve-signed" {
|
|
t.Fatalf("%+v", got)
|
|
}
|
|
if KernelSet("7.0; reboot") != nil || KernelSet("") != nil {
|
|
t.Fatal("a non-kernel string must give no set")
|
|
}
|
|
}
|
|
|
|
// ---- the clean-up of old kernels (R-1368, `09` §3 decision 286, `11` §5.11.1) ----
|
|
|
|
func cleanupRep(js string) map[string][]WrapperReport {
|
|
return map[string][]WrapperReport{"kernel-cleanup": {{KernelCleanup: json.RawMessage(js)}}}
|
|
}
|
|
|
|
// The night: the clean-up is the LAST call, after the host step, exactly once, and its plan names NO package.
|
|
// COMPANION RED-PROOF (observed): make kernelCleanupSkip return "" before its checks → the BYO / debug / unhealthy
|
|
// sub-cases of TestKernelCleanup_WhenItMayNotRun fail; put the call before runFast → this test's order check fails.
|
|
func TestKernelCleanup_NightOrderOnceAndNoNames(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: cleanupRep(`{"outcome":"removed","removed":["proxmox-kernel-7.0.14-20-pve-signed"],` +
|
|
`"kept":{"7.0.14-22-pve":["running"],"7.0.2-6-pve":["from"]},"bytes_gained":1073741824,"leftovers":[]}`)}
|
|
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
|
p := l.Run(context.Background(), 9201, "night")
|
|
cc := cleanupCalls(w)
|
|
if len(cc) != 1 {
|
|
t.Fatalf("clean-up calls = %d, want exactly one a night", len(cc))
|
|
}
|
|
if last := w.plans[len(w.plans)-1]; last["mode"] != "kernel-cleanup" {
|
|
t.Fatalf("the clean-up must be the night's last call (after the host step), last = %v", last)
|
|
}
|
|
hostAt, cleanAt := -1, -1
|
|
for i, x := range w.plans {
|
|
if x["layer"] == LayerHost {
|
|
hostAt = i
|
|
}
|
|
if x["mode"] == "kernel-cleanup" {
|
|
cleanAt = i
|
|
}
|
|
}
|
|
if hostAt < 0 || cleanAt < hostAt {
|
|
t.Fatalf("the clean-up ran before the host step: host %d clean-up %d", hostAt, cleanAt)
|
|
}
|
|
plan := cc[0]
|
|
if pk, _ := json.Marshal(plan["packages"]); string(pk) != "[]" {
|
|
t.Fatalf("the clean-up plan names packages: %s", pk)
|
|
}
|
|
for _, k := range []string{"select", "signed", "allow_new", "expect_kver", "dry_run"} {
|
|
if _, ok := plan[k]; ok {
|
|
t.Fatalf("the clean-up plan carries %q: %v", k, plan)
|
|
}
|
|
}
|
|
if plan["lane"] != "slow" || plan["layer"] != LayerKernel {
|
|
t.Fatalf("plan = %v", plan)
|
|
}
|
|
// the hub hears "cleaned" with what was removed, kept and gained — and NO kernel view / version (the hub would read
|
|
// it as a kernel step for that kernel)
|
|
rep := p.KernelCleanup
|
|
if rep.Outcome != "cleaned" || !rep.Healthy || rep.Mode != "kernel-cleanup" || rep.Layer != LayerKernel {
|
|
t.Fatalf("report = %+v", rep)
|
|
}
|
|
for _, want := range []string{"removed 1 old kernel package(s): proxmox-kernel-7.0.14-20-pve-signed", "kept 7.0.14-22-pve, 7.0.2-6-pve", "1.0 GiB gained"} {
|
|
if !strings.Contains(rep.HealthReason, want) {
|
|
t.Fatalf("report reason %q lacks %q", rep.HealthReason, want)
|
|
}
|
|
}
|
|
lastRep := h.reports[len(h.reports)-1]
|
|
if lastRep.Mode != "kernel-cleanup" || lastRep.Outcome != "cleaned" {
|
|
t.Fatalf("the hub did not hear the clean-up: %+v", lastRep)
|
|
}
|
|
if len(lastRep.Kernel) != 0 || kverRE.MatchString(lastRep.ReleaseID) {
|
|
t.Fatalf("the clean-up's report names a kernel (the hub reads that as a kernel step): %+v", lastRep)
|
|
}
|
|
// the same night again: the pass is skipped as a whole — never a second clean-up
|
|
l.Run(context.Background(), 9201, "night")
|
|
if len(cleanupCalls(w)) != 1 {
|
|
t.Fatalf("a second clean-up in the same night: %d calls", len(cleanupCalls(w)))
|
|
}
|
|
}
|
|
|
|
// "Nothing to remove" and "not run" are log lines, never a report (one a night would bury the kernel step's own line).
|
|
func TestKernelCleanup_NothingAndNotRunSendNoReport(t *testing.T) {
|
|
for name, js := range map[string]string{
|
|
"nothing": `{"outcome":"nothing","removal":[],"kept":{"7.0.14-22-pve":["running"]}}`,
|
|
"not_run": `{"outcome":"not_run","not_run":[{"code":"no-ring0-mark","reason":"ring 0 only in this release"}]}`,
|
|
} {
|
|
w := &fakeWrapper{t: t, kernelRep: cleanupRep(js)}
|
|
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
|
p := l.Run(context.Background(), 9201, "night")
|
|
if len(cleanupCalls(w)) != 1 {
|
|
t.Fatalf("%s: the wrapper was not asked", name)
|
|
}
|
|
if p.KernelCleanup.Layer != "" {
|
|
t.Fatalf("%s: a report was made: %+v", name, p.KernelCleanup)
|
|
}
|
|
for _, r := range h.reports {
|
|
if r.Mode == "kernel-cleanup" {
|
|
t.Fatalf("%s: the hub got a clean-up report: %+v", name, r)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// A refusal and a failure reach the hub (the operator), once, and are not retried.
|
|
func TestKernelCleanup_RefusedAndFailedReachTheHub(t *testing.T) {
|
|
for name, wr := range map[string]WrapperReport{
|
|
"refused": {Refused: json.RawMessage(`{"code":"P3","reason":"the pinned default and grub.cfg disagree"}`)},
|
|
"failed": {Failed: json.RawMessage(`{"rc":3,"step":"verify","reason":"the default in grub.cfg moved"}`), KernelCleanup: json.RawMessage(`{"outcome":"failed"}`)},
|
|
} {
|
|
w := &fakeWrapper{t: t, kernelRep: map[string][]WrapperReport{"kernel-cleanup": {wr}}}
|
|
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
|
p := l.Run(context.Background(), 9201, "night")
|
|
if p.KernelCleanup.Outcome != name || p.KernelCleanup.Healthy || len(p.KernelCleanup.Refused) == 0 {
|
|
t.Fatalf("%s: report = %+v", name, p.KernelCleanup)
|
|
}
|
|
if last := h.reports[len(h.reports)-1]; last.Mode != "kernel-cleanup" || last.Outcome != name {
|
|
t.Fatalf("%s: the hub did not hear it: %+v", name, last)
|
|
}
|
|
if len(cleanupCalls(w)) != 1 {
|
|
t.Fatalf("%s: retried the same night (%d calls)", name, len(cleanupCalls(w)))
|
|
}
|
|
}
|
|
}
|
|
|
|
// When it may NOT run: never with a kernel step in the night (due, staged, or reported), not in a debug pass, not on a
|
|
// BYO host, not with OS updates off, not after an unhealthy host step.
|
|
func TestKernelCleanup_WhenItMayNotRun(t *testing.T) {
|
|
type tc struct {
|
|
blk *hub.WireOSUpdate
|
|
trigger string
|
|
prep func(*fakeWrapper, *Leg)
|
|
}
|
|
bad := &Health{DockerOK: false}
|
|
cases := map[string]tc{
|
|
"a kernel step is due tonight (ring 0)": {tonight(0), "night", func(w *fakeWrapper, _ *Leg) {
|
|
w.kernelRep = map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}},
|
|
"apply": {{Upgraded: []Package{{Name: "proxmox-kernel-7.0", Version: "7.0.14-22"}}, Authority: "ring0", Kernel: kview("staged")}},
|
|
"kernel-reboot": {{Kernel: kview("oneshot")}}}
|
|
}},
|
|
"a kernel step is due tonight (ring 1, nothing staged)": {tonight(1), "night", func(w *fakeWrapper, _ *Leg) {
|
|
w.kernelRep = map[string][]WrapperReport{"kernel-status": {{Kernel: kview("none")}}}
|
|
}},
|
|
"a debug pass": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "debug", nil},
|
|
"a BYO host": {&hub.WireOSUpdate{Ring: 0, Enabled: true}, "night", func(_ *fakeWrapper, l *Leg) { l.Appliance = false }},
|
|
"updates off": {&hub.WireOSUpdate{Ring: 0, Enabled: false}, "night", nil},
|
|
"host unhealthy": {&hub.WireOSUpdate{Ring: 0, Enabled: true, HostRelease: &hub.WireOSRelease{ID: "os-host-1", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "u3", Origin: "Debian"}}}}, "night", func(w *fakeWrapper, _ *Leg) {
|
|
w.applyRep[LayerHost] = WrapperReport{Upgraded: []Package{{Name: "openssl", Version: "u3"}}, HealthAfter: &Health{HostServices: map[string]string{"pveproxy": "failed"}, GuestRunning: yes(), Guest: bad}}
|
|
w.healthSeq[LayerHost] = []*Health{{HostServices: map[string]string{"pveproxy": "failed"}, GuestRunning: yes(), Guest: bad}}
|
|
}},
|
|
}
|
|
for name, c := range cases {
|
|
w := &fakeWrapper{t: t}
|
|
l, _ := newLeg(t, w, c.blk)
|
|
if c.prep != nil {
|
|
c.prep(w, l)
|
|
}
|
|
l.HealthWait = time.Second
|
|
l.Run(context.Background(), 9201, c.trigger)
|
|
if n := len(cleanupCalls(w)); n != 0 {
|
|
t.Fatalf("%s: the clean-up ran (%d call(s)); calls = %s", name, n, calls(w))
|
|
}
|
|
}
|
|
}
|
|
|
|
// The dry run is the same mode with dry_run: true — and it sends nothing to the hub.
|
|
func TestKernelCleanup_DryRun(t *testing.T) {
|
|
w := &fakeWrapper{t: t, kernelRep: cleanupRep(`{"outcome":"dry_run","would":"remove","removal":["proxmox-kernel-7.0.14-20-pve-signed"]}`)}
|
|
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
|
raw, err := l.KernelCleanupDryRun(context.Background(), 9201)
|
|
if err != nil || !strings.Contains(string(raw), `"would":"remove"`) {
|
|
t.Fatalf("dry run = %s, %v", raw, err)
|
|
}
|
|
cc := cleanupCalls(w)
|
|
if len(cc) != 1 || cc[0]["dry_run"] != true {
|
|
t.Fatalf("the dry run's plan = %v", cc)
|
|
}
|
|
if pk, _ := json.Marshal(cc[0]["packages"]); string(pk) != "[]" {
|
|
t.Fatalf("the dry run's plan names packages: %s", pk)
|
|
}
|
|
if len(h.reports) != 0 {
|
|
t.Fatalf("a dry run reported to the hub: %+v", h.reports)
|
|
}
|
|
}
|