package osupdates import ( "encoding/json" "errors" "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // R-836, `09` §3 decision 172: the kernel lane's hub half — due boxes, the day-before mail (no mail, no step), the // block, the events, the operator's approval. const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve" // laneFacts stores a host report whose system facts carry the kernel lane. func (f *fix) laneFacts(t *testing.T, host, cust, running, phase, to string, problems ...string) { t.Helper() lane := map[string]any{"running": running, "default": running, "phase": phase, "to": to, "from": kOld} if len(problems) > 0 { lane["setup_problems"] = problems } body, _ := json.Marshal(map[string]any{"system": map[string]any{"facts": map[string]any{"host": map[string]any{ "kernel_running": running, "kernel_lane": lane}}}}) if err := f.s.Store.SaveHostReport(host, cust, body, store.HostReportDenorm{AgentVersion: "0.152.0"}); err != nil { t.Fatal(err) } } // hostWithPendingKernel reports a host step whose pending list carries the kernel meta-package upgrade. func (f *fix) hostWithPendingKernel(t *testing.T, host string) { f.ingest(t, host, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: []PendingPkg{{Name: "proxmox-kernel-7.0", From: "7.0.2-6", To: "7.0.14-22", Origin: []string{"Proxmox Debian Repository"}}, {Name: "pve-manager", From: "9.2.2", To: "9.2.21", Origin: []string{"Proxmox Debian Repository"}}}}) } func kreport(outcome, trigger string, healthy bool) Report { return Report{Layer: LayerKernel, Trigger: trigger, Mode: "apply", Outcome: outcome, Healthy: healthy, ReleaseID: kNew, Kernel: json.RawMessage(`{"from":"` + kOld + `","to":"` + kNew + `"}`)} } func (f *fix) mail(sent *[]string, fail error) { f.s.KernelMail = func(cust, kver string) (string, error) { if fail != nil { return "", fail } *sent = append(*sent, cust+":"+kver) return "hu", nil } } func budapest(t *testing.T, h int) time.Time { loc, err := time.LoadLocation("Europe/Budapest") if err != nil { t.Skip("no tzdata") } return time.Date(2026, 10, 8, h, 30, 0, 0, loc) } // A ring-0 box with a pending kernel is due; its block names the kernel, NOT tonight, until the household was mailed. // COMPANION RED-PROOF (observed): set Tonight without a notice in KernelBlockFor → "tonight before any mail". func TestKernel_DueButNotToldIsNotTonight(t *testing.T) { f := newFix(t) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "none", "") k, why := f.s.KernelDue("hp") if k != kNew { t.Fatalf("due = %q (%s)", k, why) } b := f.s.DesiredBlock("hp").Kernel if b == nil || b.Kver != kNew || b.Tonight { t.Fatalf("tonight before any mail: %+v", b) } } // The mail goes out only 09:00–20:00 Budapest; then the block says tonight (for 24 h); never two within 20 h. func TestKernel_DayBeforeMailMakesTonight(t *testing.T) { f := newFix(t) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "none", "") var sent []string f.mail(&sent, nil) f.now = budapest(t, 3) // night: no mail if told := f.s.KernelNotify(); len(told) != 0 || len(sent) != 0 { t.Fatalf("mailed at 03:30: %v", sent) } f.now = budapest(t, 21) if f.s.KernelNotify(); len(sent) != 0 { t.Fatalf("mailed at 21:30: %v", sent) } f.now = budapest(t, 10) if told := f.s.KernelNotify(); len(told) != 1 || len(sent) != 1 || sent[0] != "c-hp:"+kNew { t.Fatalf("told=%v sent=%v", told, sent) } if b := f.s.DesiredBlock("hp").Kernel; b == nil || !b.Tonight || b.NotifiedAt == "" { t.Fatalf("after the mail the block must say tonight: %+v", b) } if !contains(f.bumps, "hp") || !contains(f.events, EventKernelNotice) { t.Fatalf("the box must be bumped and the operator told: bumps=%v events=%v", f.bumps, f.events) } f.now = f.now.Add(5 * time.Hour) if f.s.KernelNotify(); len(sent) != 1 { t.Fatalf("a second mail within 20 h: %v", sent) } f.now = budapest(t, 10).Add(25 * time.Hour) if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight { t.Fatalf("a mail older than 24 h no longer covers tonight: %+v", b) } } // No mail → no step: a mail the service did not accept is not recorded; the block stays not-tonight; the operator hears. // COMPANION RED-PROOF (observed): record the notice before checking the send error → "a failed mail made tonight". func TestKernel_NoMailNoStep(t *testing.T) { f := newFix(t) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "none", "") var sent []string f.mail(&sent, errors.New("the household has no e-mail address")) f.now = budapest(t, 11) f.s.KernelNotify() if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight { t.Fatalf("a failed mail made tonight: %+v", b) } if !contains(f.events, EventKernelNotice) { t.Fatalf("the operator must hear the household could not be told: %v", f.events) } f.s.KernelMail = nil f.s.KernelNotify() if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight { t.Fatalf("no mailer → never tonight: %+v", b) } } // An ended step is never retried by itself; the operator decides. func TestKernel_EndedStepIsNeverRetried(t *testing.T) { for _, out := range []string{"applied", "fell_back", "health_failed", "self_reverted", "revert_failed"} { f := newFix(t) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "fell_back", kNew) f.ingest(t, "hp", kreport(out, "boot", out == "applied")) if k, why := f.s.KernelDue("hp"); k != "" || !strings.Contains(why, "operator decides") { t.Fatalf("%s: still due (%q %q)", out, k, why) } if f.s.DesiredBlock("hp").Kernel != nil { t.Fatalf("%s: block still names a kernel", out) } } // a refusal for good ends it; a passing one (a lock) does not f := newFix(t) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "none", "") r := kreport("refused", "night", false) r.Refused = json.RawMessage(`{"code":"R9","reason":"another apt holds the lock"}`) f.ingest(t, "hp", r) if k, _ := f.s.KernelDue("hp"); k != kNew { t.Fatal("a transient refusal (R9) must not end the step") } r.Refused = json.RawMessage(`{"code":"R20","reason":"/boot/efi is not vfat"}`) f.ingest(t, "hp", r) if k, _ := f.s.KernelDue("hp"); k != "" { t.Fatal("R20 must end the step") } } // Ring 1 is due ONLY for a kernel a signed job staged; a box with a setup problem or switched off never is. func TestKernel_Ring1OnlyWhenStagedAndSetupMustBeFine(t *testing.T) { f := newFix(t) f.hostWithPendingKernel(t, "cust1") f.laneFacts(t, "cust1", "c-1", kOld, "none", "") if k, _ := f.s.KernelDue("cust1"); k != "" { t.Fatal("ring 1 due without a staged kernel") } f.laneFacts(t, "cust1", "c-1", kOld, "staged", kNew) if k, why := f.s.KernelDue("cust1"); k != kNew { t.Fatalf("ring 1 with a staged kernel not due: %s", why) } f.laneFacts(t, "cust1", "c-1", kOld, "staged", kNew, "/boot/efi is not a mounted vfat ESP") if k, _ := f.s.KernelDue("cust1"); k != "" { t.Fatal("a box that cannot do a one-shot is never due") } _ = f.s.Store.SetOSEnabled("hp", false) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "none", "") if k, _ := f.s.KernelDue("hp"); k != "" { t.Fatal("switched off but due") } } // The household is told at most KernelNoticeMax times for one kernel; then the operator hears once. func TestKernel_AtMostThreeMails(t *testing.T) { f := newFix(t) f.hostWithPendingKernel(t, "hp") f.laneFacts(t, "hp", "c-hp", kOld, "none", "") var sent []string f.mail(&sent, nil) day := budapest(t, 10) for i := 0; i < 5; i++ { f.now = day.Add(time.Duration(i) * 24 * time.Hour) f.s.KernelNotify() } if len(sent) != KernelNoticeMax { t.Fatalf("mails = %d, want %d", len(sent), KernelNoticeMax) } n := 0 for _, e := range f.events { if e == EventKernelNotice { n++ } } if n != KernelNoticeMax+1 { t.Fatalf("notice events = %d (3 told + 1 'no further mail')", n) } } // Each outcome is an operator event; "applied" also gives the household its timeline line; ingest never treats a // kernel report as a guest report. func TestKernel_IngestEvents(t *testing.T) { f := newFix(t) f.ingest(t, "hp", kreport("staged", "night", true)) f.ingest(t, "hp", kreport("applied", "boot", true)) f.ingest(t, "hp", kreport("fell_back", "boot", false)) if c := countOf(f.events, EventKernelStep); c != 3 { t.Fatalf("kernel events = %d: %v", c, f.events) } if !contains(f.events, EventApplied) { t.Fatal("an applied kernel gives the household its line") } if rep, _ := f.s.Store.LatestOSReport("hp", LayerGuest); rep != nil { t.Fatalf("a kernel report was stored as a guest report: %+v", rep) } if rep, _ := f.s.Store.LatestOSReport("hp", LayerKernel); rep == nil || rep.Outcome != "fell_back" { t.Fatalf("kernel report not stored on its layer: %+v", rep) } } // The operator's approval: every ring-0 box booted the same kernel healthily after a night stage — never before. // COMPANION RED-PROOF (observed): drop the `night` check in KernelStatus → "approved without a night stage". func TestKernel_ApproveNeedsEveryRing0BoxHealthyAfterANightStep(t *testing.T) { f := newFix(t) if _, err := f.s.ApproveKernel(); err == nil { t.Fatal("approved with nothing booted") } f.ingest(t, "hp", kreport("staged", "night", true)) f.ingest(t, "hp", kreport("applied", "boot", true)) f.ingest(t, "n100", kreport("staged", "signed", true)) // a by-day stage is not a night step f.ingest(t, "n100", kreport("applied", "boot", true)) if _, err := f.s.ApproveKernel(); err == nil || !strings.Contains(err.Error(), "night") { t.Fatalf("approved without a night stage: %v", err) } f.ingest(t, "n100", kreport("staged", "night", true)) f.ingest(t, "n100", kreport("applied", "boot", true)) id, err := f.s.ApproveKernel() if err != nil || !strings.HasPrefix(id, "os-kernel-") { t.Fatalf("%q %v", id, err) } rel, _ := f.s.Store.LatestOSRelease(LayerKernel) if !strings.Contains(rel.PackagesJSON, `"proxmox-kernel-7.0"`) || !strings.Contains(rel.PackagesJSON, "proxmox-kernel-7.0.14-22-pve-signed") { t.Fatalf("release = %s", rel.PackagesJSON) } if contains(f.bumps, "cust1") { t.Fatal("an approved kernel set must nudge no ring-1 box (a signed job only)") } // a fell-back ring-0 box blocks the button g := newFix(t) g.ingest(t, "hp", kreport("staged", "night", true)) g.ingest(t, "hp", kreport("fell_back", "boot", false)) if st, _ := g.s.KernelStatus(); !strings.Contains(st.Waiting, "fell_back") { t.Fatalf("waiting = %q", st.Waiting) } } func contains(xs []string, x string) bool { return countOf(xs, x) > 0 } func countOf(xs []string, x string) int { n := 0 for _, v := range xs { if v == x { n++ } } return n } // R-898: a told kernel that can no longer be installed is refused by the box BEFORE any change (R7) — temporary, never // "ended for good": the box stays due, and once its report offers the newer kernel the household is told again (inside // the 3-mail limit). COMPANION RED-PROOF (observed): add "R7" to kernelRefusedForGood → "an R7 refusal ended the step". func TestKernel_ToldKernelGoneIsTemporaryAndTheNewerIsOffered(t *testing.T) { f := newFix(t) f.hostWithPendingKernel(t, "hp") // pending: 7.0.14-22 f.laneFacts(t, "hp", "c-hp", kOld, "none", "") r := kreport("refused", "night", false) r.Refused = json.RawMessage(`{"code":"R7","reason":"the simulation failed: E: Version '7.0.14-22' for 'proxmox-kernel-7.0' was not found"}`) f.ingest(t, "hp", r) if k, why := f.s.KernelDue("hp"); k != kNew { t.Fatalf("an R7 refusal ended the step: %q %q", k, why) } // the next night's report offers a newer kernel → that one is due, and its household mail is a new one f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: []PendingPkg{{Name: "proxmox-kernel-7.0", From: "7.0.2-6", To: "7.0.14-23", Origin: []string{"Proxmox Debian Repository"}}}}) if k, _ := f.s.KernelDue("hp"); k != "7.0.14-23-pve" { t.Fatalf("the newer kernel is not offered: %q", k) } var sent []string f.mail(&sent, nil) f.now = budapest(t, 10) if f.s.KernelNotify(); len(sent) != 1 || sent[0] != "c-hp:7.0.14-23-pve" { t.Fatalf("the household must be told again for the newer kernel: %v", sent) } }