admin
87765bfa63
gates / gates (push) Successful in 3m21s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
312 lines
12 KiB
Go
312 lines
12 KiB
Go
package osupdates
|
||
|
||
import (
|
||
"encoding/json"
|
||
"errors"
|
||
"strings"
|
||
"testing"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
||
)
|
||
|
||
// R-836, `09` §3 decision 172: the kernel lane's hub half — due boxes, the day-before mail (no mail, no step), the
|
||
// block, the events, the operator's approval.
|
||
|
||
const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve"
|
||
|
||
// laneFacts stores a host report whose system facts carry the kernel lane.
|
||
func (f *fix) laneFacts(t *testing.T, host, cust, running, phase, to string, problems ...string) {
|
||
t.Helper()
|
||
lane := map[string]any{"running": running, "default": running, "phase": phase, "to": to, "from": kOld}
|
||
if len(problems) > 0 {
|
||
lane["setup_problems"] = problems
|
||
}
|
||
body, _ := json.Marshal(map[string]any{"system": map[string]any{"facts": map[string]any{"host": map[string]any{
|
||
"kernel_running": running, "kernel_lane": lane}}}})
|
||
if err := f.s.Store.SaveHostReport(host, cust, body, store.HostReportDenorm{AgentVersion: "0.152.0"}); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
}
|
||
|
||
// hostWithPendingKernel reports a host step whose pending list carries the kernel meta-package upgrade.
|
||
func (f *fix) hostWithPendingKernel(t *testing.T, host string) {
|
||
f.ingest(t, host, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true,
|
||
Pending: []PendingPkg{{Name: "proxmox-kernel-7.0", From: "7.0.2-6", To: "7.0.14-22", Origin: []string{"Proxmox Debian Repository"}},
|
||
{Name: "pve-manager", From: "9.2.2", To: "9.2.21", Origin: []string{"Proxmox Debian Repository"}}}})
|
||
}
|
||
|
||
func kreport(outcome, trigger string, healthy bool) Report {
|
||
return Report{Layer: LayerKernel, Trigger: trigger, Mode: "apply", Outcome: outcome, Healthy: healthy, ReleaseID: kNew,
|
||
Kernel: json.RawMessage(`{"from":"` + kOld + `","to":"` + kNew + `"}`)}
|
||
}
|
||
|
||
func (f *fix) mail(sent *[]string, fail error) {
|
||
f.s.KernelMail = func(cust, kver string) (string, error) {
|
||
if fail != nil {
|
||
return "", fail
|
||
}
|
||
*sent = append(*sent, cust+":"+kver)
|
||
return "hu", nil
|
||
}
|
||
}
|
||
|
||
func budapest(t *testing.T, h int) time.Time {
|
||
loc, err := time.LoadLocation("Europe/Budapest")
|
||
if err != nil {
|
||
t.Skip("no tzdata")
|
||
}
|
||
return time.Date(2026, 10, 8, h, 30, 0, 0, loc)
|
||
}
|
||
|
||
// A ring-0 box with a pending kernel is due; its block names the kernel, NOT tonight, until the household was mailed.
|
||
// COMPANION RED-PROOF (observed): set Tonight without a notice in KernelBlockFor → "tonight before any mail".
|
||
func TestKernel_DueButNotToldIsNotTonight(t *testing.T) {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
k, why := f.s.KernelDue("hp")
|
||
if k != kNew {
|
||
t.Fatalf("due = %q (%s)", k, why)
|
||
}
|
||
b := f.s.DesiredBlock("hp").Kernel
|
||
if b == nil || b.Kver != kNew || b.Tonight {
|
||
t.Fatalf("tonight before any mail: %+v", b)
|
||
}
|
||
}
|
||
|
||
// The mail goes out only 09:00–20:00 Budapest; then the block says tonight (for 24 h); never two within 20 h.
|
||
func TestKernel_DayBeforeMailMakesTonight(t *testing.T) {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
var sent []string
|
||
f.mail(&sent, nil)
|
||
f.now = budapest(t, 3) // night: no mail
|
||
if told := f.s.KernelNotify(); len(told) != 0 || len(sent) != 0 {
|
||
t.Fatalf("mailed at 03:30: %v", sent)
|
||
}
|
||
f.now = budapest(t, 21)
|
||
if f.s.KernelNotify(); len(sent) != 0 {
|
||
t.Fatalf("mailed at 21:30: %v", sent)
|
||
}
|
||
f.now = budapest(t, 10)
|
||
if told := f.s.KernelNotify(); len(told) != 1 || len(sent) != 1 || sent[0] != "c-hp:"+kNew {
|
||
t.Fatalf("told=%v sent=%v", told, sent)
|
||
}
|
||
if b := f.s.DesiredBlock("hp").Kernel; b == nil || !b.Tonight || b.NotifiedAt == "" {
|
||
t.Fatalf("after the mail the block must say tonight: %+v", b)
|
||
}
|
||
if !contains(f.bumps, "hp") || !contains(f.events, EventKernelNotice) {
|
||
t.Fatalf("the box must be bumped and the operator told: bumps=%v events=%v", f.bumps, f.events)
|
||
}
|
||
f.now = f.now.Add(5 * time.Hour)
|
||
if f.s.KernelNotify(); len(sent) != 1 {
|
||
t.Fatalf("a second mail within 20 h: %v", sent)
|
||
}
|
||
f.now = budapest(t, 10).Add(25 * time.Hour)
|
||
if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight {
|
||
t.Fatalf("a mail older than 24 h no longer covers tonight: %+v", b)
|
||
}
|
||
}
|
||
|
||
// No mail → no step: a mail the service did not accept is not recorded; the block stays not-tonight; the operator hears.
|
||
// COMPANION RED-PROOF (observed): record the notice before checking the send error → "a failed mail made tonight".
|
||
func TestKernel_NoMailNoStep(t *testing.T) {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
var sent []string
|
||
f.mail(&sent, errors.New("the household has no e-mail address"))
|
||
f.now = budapest(t, 11)
|
||
f.s.KernelNotify()
|
||
if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight {
|
||
t.Fatalf("a failed mail made tonight: %+v", b)
|
||
}
|
||
if !contains(f.events, EventKernelNotice) {
|
||
t.Fatalf("the operator must hear the household could not be told: %v", f.events)
|
||
}
|
||
f.s.KernelMail = nil
|
||
f.s.KernelNotify()
|
||
if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight {
|
||
t.Fatalf("no mailer → never tonight: %+v", b)
|
||
}
|
||
}
|
||
|
||
// An ended step is never retried by itself; the operator decides.
|
||
func TestKernel_EndedStepIsNeverRetried(t *testing.T) {
|
||
for _, out := range []string{"applied", "fell_back", "health_failed", "self_reverted", "revert_failed"} {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "fell_back", kNew)
|
||
f.ingest(t, "hp", kreport(out, "boot", out == "applied"))
|
||
if k, why := f.s.KernelDue("hp"); k != "" || !strings.Contains(why, "operator decides") {
|
||
t.Fatalf("%s: still due (%q %q)", out, k, why)
|
||
}
|
||
if f.s.DesiredBlock("hp").Kernel != nil {
|
||
t.Fatalf("%s: block still names a kernel", out)
|
||
}
|
||
}
|
||
// a refusal for good ends it; a passing one (a lock) does not
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
r := kreport("refused", "night", false)
|
||
r.Refused = json.RawMessage(`{"code":"R9","reason":"another apt holds the lock"}`)
|
||
f.ingest(t, "hp", r)
|
||
if k, _ := f.s.KernelDue("hp"); k != kNew {
|
||
t.Fatal("a transient refusal (R9) must not end the step")
|
||
}
|
||
r.Refused = json.RawMessage(`{"code":"R20","reason":"/boot/efi is not vfat"}`)
|
||
f.ingest(t, "hp", r)
|
||
if k, _ := f.s.KernelDue("hp"); k != "" {
|
||
t.Fatal("R20 must end the step")
|
||
}
|
||
}
|
||
|
||
// Ring 1 is due ONLY for a kernel a signed job staged; a box with a setup problem or switched off never is.
|
||
func TestKernel_Ring1OnlyWhenStagedAndSetupMustBeFine(t *testing.T) {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "cust1")
|
||
f.laneFacts(t, "cust1", "c-1", kOld, "none", "")
|
||
if k, _ := f.s.KernelDue("cust1"); k != "" {
|
||
t.Fatal("ring 1 due without a staged kernel")
|
||
}
|
||
f.laneFacts(t, "cust1", "c-1", kOld, "staged", kNew)
|
||
if k, why := f.s.KernelDue("cust1"); k != kNew {
|
||
t.Fatalf("ring 1 with a staged kernel not due: %s", why)
|
||
}
|
||
f.laneFacts(t, "cust1", "c-1", kOld, "staged", kNew, "/boot/efi is not a mounted vfat ESP")
|
||
if k, _ := f.s.KernelDue("cust1"); k != "" {
|
||
t.Fatal("a box that cannot do a one-shot is never due")
|
||
}
|
||
_ = f.s.Store.SetOSEnabled("hp", false)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
if k, _ := f.s.KernelDue("hp"); k != "" {
|
||
t.Fatal("switched off but due")
|
||
}
|
||
}
|
||
|
||
// The household is told at most KernelNoticeMax times for one kernel; then the operator hears once.
|
||
func TestKernel_AtMostThreeMails(t *testing.T) {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp")
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
var sent []string
|
||
f.mail(&sent, nil)
|
||
day := budapest(t, 10)
|
||
for i := 0; i < 5; i++ {
|
||
f.now = day.Add(time.Duration(i) * 24 * time.Hour)
|
||
f.s.KernelNotify()
|
||
}
|
||
if len(sent) != KernelNoticeMax {
|
||
t.Fatalf("mails = %d, want %d", len(sent), KernelNoticeMax)
|
||
}
|
||
n := 0
|
||
for _, e := range f.events {
|
||
if e == EventKernelNotice {
|
||
n++
|
||
}
|
||
}
|
||
if n != KernelNoticeMax+1 {
|
||
t.Fatalf("notice events = %d (3 told + 1 'no further mail')", n)
|
||
}
|
||
}
|
||
|
||
// Each outcome is an operator event; "applied" also gives the household its timeline line; ingest never treats a
|
||
// kernel report as a guest report.
|
||
func TestKernel_IngestEvents(t *testing.T) {
|
||
f := newFix(t)
|
||
f.ingest(t, "hp", kreport("staged", "night", true))
|
||
f.ingest(t, "hp", kreport("applied", "boot", true))
|
||
f.ingest(t, "hp", kreport("fell_back", "boot", false))
|
||
if c := countOf(f.events, EventKernelStep); c != 3 {
|
||
t.Fatalf("kernel events = %d: %v", c, f.events)
|
||
}
|
||
if !contains(f.events, EventApplied) {
|
||
t.Fatal("an applied kernel gives the household its line")
|
||
}
|
||
if rep, _ := f.s.Store.LatestOSReport("hp", LayerGuest); rep != nil {
|
||
t.Fatalf("a kernel report was stored as a guest report: %+v", rep)
|
||
}
|
||
if rep, _ := f.s.Store.LatestOSReport("hp", LayerKernel); rep == nil || rep.Outcome != "fell_back" {
|
||
t.Fatalf("kernel report not stored on its layer: %+v", rep)
|
||
}
|
||
}
|
||
|
||
// The operator's approval: every ring-0 box booted the same kernel healthily after a night stage — never before.
|
||
// COMPANION RED-PROOF (observed): drop the `night` check in KernelStatus → "approved without a night stage".
|
||
func TestKernel_ApproveNeedsEveryRing0BoxHealthyAfterANightStep(t *testing.T) {
|
||
f := newFix(t)
|
||
if _, err := f.s.ApproveKernel(); err == nil {
|
||
t.Fatal("approved with nothing booted")
|
||
}
|
||
f.ingest(t, "hp", kreport("staged", "night", true))
|
||
f.ingest(t, "hp", kreport("applied", "boot", true))
|
||
f.ingest(t, "n100", kreport("staged", "signed", true)) // a by-day stage is not a night step
|
||
f.ingest(t, "n100", kreport("applied", "boot", true))
|
||
if _, err := f.s.ApproveKernel(); err == nil || !strings.Contains(err.Error(), "night") {
|
||
t.Fatalf("approved without a night stage: %v", err)
|
||
}
|
||
f.ingest(t, "n100", kreport("staged", "night", true))
|
||
f.ingest(t, "n100", kreport("applied", "boot", true))
|
||
id, err := f.s.ApproveKernel()
|
||
if err != nil || !strings.HasPrefix(id, "os-kernel-") {
|
||
t.Fatalf("%q %v", id, err)
|
||
}
|
||
rel, _ := f.s.Store.LatestOSRelease(LayerKernel)
|
||
if !strings.Contains(rel.PackagesJSON, `"proxmox-kernel-7.0"`) || !strings.Contains(rel.PackagesJSON, "proxmox-kernel-7.0.14-22-pve-signed") {
|
||
t.Fatalf("release = %s", rel.PackagesJSON)
|
||
}
|
||
if contains(f.bumps, "cust1") {
|
||
t.Fatal("an approved kernel set must nudge no ring-1 box (a signed job only)")
|
||
}
|
||
// a fell-back ring-0 box blocks the button
|
||
g := newFix(t)
|
||
g.ingest(t, "hp", kreport("staged", "night", true))
|
||
g.ingest(t, "hp", kreport("fell_back", "boot", false))
|
||
if st, _ := g.s.KernelStatus(); !strings.Contains(st.Waiting, "fell_back") {
|
||
t.Fatalf("waiting = %q", st.Waiting)
|
||
}
|
||
}
|
||
|
||
func contains(xs []string, x string) bool { return countOf(xs, x) > 0 }
|
||
|
||
func countOf(xs []string, x string) int {
|
||
n := 0
|
||
for _, v := range xs {
|
||
if v == x {
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
// R-898: a told kernel that can no longer be installed is refused by the box BEFORE any change (R7) — temporary, never
|
||
// "ended for good": the box stays due, and once its report offers the newer kernel the household is told again (inside
|
||
// the 3-mail limit). COMPANION RED-PROOF (observed): add "R7" to kernelRefusedForGood → "an R7 refusal ended the step".
|
||
func TestKernel_ToldKernelGoneIsTemporaryAndTheNewerIsOffered(t *testing.T) {
|
||
f := newFix(t)
|
||
f.hostWithPendingKernel(t, "hp") // pending: 7.0.14-22
|
||
f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
|
||
r := kreport("refused", "night", false)
|
||
r.Refused = json.RawMessage(`{"code":"R7","reason":"the simulation failed: E: Version '7.0.14-22' for 'proxmox-kernel-7.0' was not found"}`)
|
||
f.ingest(t, "hp", r)
|
||
if k, why := f.s.KernelDue("hp"); k != kNew {
|
||
t.Fatalf("an R7 refusal ended the step: %q %q", k, why)
|
||
}
|
||
// the next night's report offers a newer kernel → that one is due, and its household mail is a new one
|
||
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true,
|
||
Pending: []PendingPkg{{Name: "proxmox-kernel-7.0", From: "7.0.2-6", To: "7.0.14-23", Origin: []string{"Proxmox Debian Repository"}}}})
|
||
if k, _ := f.s.KernelDue("hp"); k != "7.0.14-23-pve" {
|
||
t.Fatalf("the newer kernel is not offered: %q", k)
|
||
}
|
||
var sent []string
|
||
f.mail(&sent, nil)
|
||
f.now = budapest(t, 10)
|
||
if f.s.KernelNotify(); len(sent) != 1 || sent[0] != "c-hp:7.0.14-23-pve" {
|
||
t.Fatalf("the household must be told again for the newer kernel: %v", sent)
|
||
}
|
||
}
|