felhom.eu/hub/internal/osupdates/kernel_test.go

package osupdates

import (
	"encoding/json"
	"errors"
	"strings"
	"testing"
	"time"

	"gitea.dooplex.hu/admin/felhom-hub/internal/store"
)

// R-836, `09` §3 decision 172: the kernel lane's hub half — due boxes, the day-before mail (no mail, no step), the
// block, the events, the operator's approval.

const kOld, kNew = "7.0.2-6-pve", "7.0.14-22-pve"

// laneFacts stores a host report whose system facts carry the kernel lane.
func (f *fix) laneFacts(t *testing.T, host, cust, running, phase, to string, problems ...string) {
	t.Helper()
	lane := map[string]any{"running": running, "default": running, "phase": phase, "to": to, "from": kOld}
	if len(problems) > 0 {
		lane["setup_problems"] = problems
	}
	body, _ := json.Marshal(map[string]any{"system": map[string]any{"facts": map[string]any{"host": map[string]any{
		"kernel_running": running, "kernel_lane": lane}}}})
	if err := f.s.Store.SaveHostReport(host, cust, body, store.HostReportDenorm{AgentVersion: "0.152.0"}); err != nil {
		t.Fatal(err)
	}
}

// hostWithPendingKernel reports a host step whose pending list carries the kernel meta-package upgrade.
func (f *fix) hostWithPendingKernel(t *testing.T, host string) {
	f.ingest(t, host, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true,
		Pending: []PendingPkg{{Name: "proxmox-kernel-7.0", From: "7.0.2-6", To: "7.0.14-22", Origin: []string{"Proxmox Debian Repository"}},
			{Name: "pve-manager", From: "9.2.2", To: "9.2.21", Origin: []string{"Proxmox Debian Repository"}}}})
}

func kreport(outcome, trigger string, healthy bool) Report {
	return Report{Layer: LayerKernel, Trigger: trigger, Mode: "apply", Outcome: outcome, Healthy: healthy, ReleaseID: kNew,
		Kernel: json.RawMessage(`{"from":"` + kOld + `","to":"` + kNew + `"}`)}
}

func (f *fix) mail(sent *[]string, fail error) {
	f.s.KernelMail = func(cust, kver string) (string, error) {
		if fail != nil {
			return "", fail
		}
		*sent = append(*sent, cust+":"+kver)
		return "hu", nil
	}
}

func budapest(t *testing.T, h int) time.Time {
	loc, err := time.LoadLocation("Europe/Budapest")
	if err != nil {
		t.Skip("no tzdata")
	}
	return time.Date(2026, 10, 8, h, 30, 0, 0, loc)
}

// A ring-0 box with a pending kernel is due; its block names the kernel, NOT tonight, until the household was mailed.
// COMPANION RED-PROOF (observed): set Tonight without a notice in KernelBlockFor → "tonight before any mail".
func TestKernel_DueButNotToldIsNotTonight(t *testing.T) {
	f := newFix(t)
	f.hostWithPendingKernel(t, "hp")
	f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
	k, why := f.s.KernelDue("hp")
	if k != kNew {
		t.Fatalf("due = %q (%s)", k, why)
	}
	b := f.s.DesiredBlock("hp").Kernel
	if b == nil || b.Kver != kNew || b.Tonight {
		t.Fatalf("tonight before any mail: %+v", b)
	}
}

// The mail goes out only 09:00–20:00 Budapest; then the block says tonight (for 24 h); never two within 20 h.
func TestKernel_DayBeforeMailMakesTonight(t *testing.T) {
	f := newFix(t)
	f.hostWithPendingKernel(t, "hp")
	f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
	var sent []string
	f.mail(&sent, nil)
	f.now = budapest(t, 3) // night: no mail
	if told := f.s.KernelNotify(); len(told) != 0 || len(sent) != 0 {
		t.Fatalf("mailed at 03:30: %v", sent)
	}
	f.now = budapest(t, 21)
	if f.s.KernelNotify(); len(sent) != 0 {
		t.Fatalf("mailed at 21:30: %v", sent)
	}
	f.now = budapest(t, 10)
	if told := f.s.KernelNotify(); len(told) != 1 || len(sent) != 1 || sent[0] != "c-hp:"+kNew {
		t.Fatalf("told=%v sent=%v", told, sent)
	}
	if b := f.s.DesiredBlock("hp").Kernel; b == nil || !b.Tonight || b.NotifiedAt == "" {
		t.Fatalf("after the mail the block must say tonight: %+v", b)
	}
	if !contains(f.bumps, "hp") || !contains(f.events, EventKernelNotice) {
		t.Fatalf("the box must be bumped and the operator told: bumps=%v events=%v", f.bumps, f.events)
	}
	f.now = f.now.Add(5 * time.Hour)
	if f.s.KernelNotify(); len(sent) != 1 {
		t.Fatalf("a second mail within 20 h: %v", sent)
	}
	f.now = budapest(t, 10).Add(25 * time.Hour)
	if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight {
		t.Fatalf("a mail older than 24 h no longer covers tonight: %+v", b)
	}
}

// No mail → no step: a mail the service did not accept is not recorded; the block stays not-tonight; the operator hears.
// COMPANION RED-PROOF (observed): record the notice before checking the send error → "a failed mail made tonight".
func TestKernel_NoMailNoStep(t *testing.T) {
	f := newFix(t)
	f.hostWithPendingKernel(t, "hp")
	f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
	var sent []string
	f.mail(&sent, errors.New("the household has no e-mail address"))
	f.now = budapest(t, 11)
	f.s.KernelNotify()
	if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight {
		t.Fatalf("a failed mail made tonight: %+v", b)
	}
	if !contains(f.events, EventKernelNotice) {
		t.Fatalf("the operator must hear the household could not be told: %v", f.events)
	}
	f.s.KernelMail = nil
	f.s.KernelNotify()
	if b := f.s.DesiredBlock("hp").Kernel; b == nil || b.Tonight {
		t.Fatalf("no mailer → never tonight: %+v", b)
	}
}

// An ended step is never retried by itself; the operator decides.
func TestKernel_EndedStepIsNeverRetried(t *testing.T) {
	for _, out := range []string{"applied", "fell_back", "health_failed", "self_reverted", "revert_failed"} {
		f := newFix(t)
		f.hostWithPendingKernel(t, "hp")
		f.laneFacts(t, "hp", "c-hp", kOld, "fell_back", kNew)
		f.ingest(t, "hp", kreport(out, "boot", out == "applied"))
		if k, why := f.s.KernelDue("hp"); k != "" || !strings.Contains(why, "operator decides") {
			t.Fatalf("%s: still due (%q %q)", out, k, why)
		}
		if f.s.DesiredBlock("hp").Kernel != nil {
			t.Fatalf("%s: block still names a kernel", out)
		}
	}
	// a refusal for good ends it; a passing one (a lock) does not
	f := newFix(t)
	f.hostWithPendingKernel(t, "hp")
	f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
	r := kreport("refused", "night", false)
	r.Refused = json.RawMessage(`{"code":"R9","reason":"another apt holds the lock"}`)
	f.ingest(t, "hp", r)
	if k, _ := f.s.KernelDue("hp"); k != kNew {
		t.Fatal("a transient refusal (R9) must not end the step")
	}
	r.Refused = json.RawMessage(`{"code":"R20","reason":"/boot/efi is not vfat"}`)
	f.ingest(t, "hp", r)
	if k, _ := f.s.KernelDue("hp"); k != "" {
		t.Fatal("R20 must end the step")
	}
}

// Ring 1 is due ONLY for a kernel a signed job staged; a box with a setup problem or switched off never is.
func TestKernel_Ring1OnlyWhenStagedAndSetupMustBeFine(t *testing.T) {
	f := newFix(t)
	f.hostWithPendingKernel(t, "cust1")
	f.laneFacts(t, "cust1", "c-1", kOld, "none", "")
	if k, _ := f.s.KernelDue("cust1"); k != "" {
		t.Fatal("ring 1 due without a staged kernel")
	}
	f.laneFacts(t, "cust1", "c-1", kOld, "staged", kNew)
	if k, why := f.s.KernelDue("cust1"); k != kNew {
		t.Fatalf("ring 1 with a staged kernel not due: %s", why)
	}
	f.laneFacts(t, "cust1", "c-1", kOld, "staged", kNew, "/boot/efi is not a mounted vfat ESP")
	if k, _ := f.s.KernelDue("cust1"); k != "" {
		t.Fatal("a box that cannot do a one-shot is never due")
	}
	_ = f.s.Store.SetOSEnabled("hp", false)
	f.hostWithPendingKernel(t, "hp")
	f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
	if k, _ := f.s.KernelDue("hp"); k != "" {
		t.Fatal("switched off but due")
	}
}

// The household is told at most KernelNoticeMax times for one kernel; then the operator hears once.
func TestKernel_AtMostThreeMails(t *testing.T) {
	f := newFix(t)
	f.hostWithPendingKernel(t, "hp")
	f.laneFacts(t, "hp", "c-hp", kOld, "none", "")
	var sent []string
	f.mail(&sent, nil)
	day := budapest(t, 10)
	for i := 0; i < 5; i++ {
		f.now = day.Add(time.Duration(i) * 24 * time.Hour)
		f.s.KernelNotify()
	}
	if len(sent) != KernelNoticeMax {
		t.Fatalf("mails = %d, want %d", len(sent), KernelNoticeMax)
	}
	n := 0
	for _, e := range f.events {
		if e == EventKernelNotice {
			n++
		}
	}
	if n != KernelNoticeMax+1 {
		t.Fatalf("notice events = %d (3 told + 1 'no further mail')", n)
	}
}

// Each outcome is an operator event; "applied" also gives the household its timeline line; ingest never treats a
// kernel report as a guest report.
func TestKernel_IngestEvents(t *testing.T) {
	f := newFix(t)
	f.ingest(t, "hp", kreport("staged", "night", true))
	f.ingest(t, "hp", kreport("applied", "boot", true))
	f.ingest(t, "hp", kreport("fell_back", "boot", false))
	if c := countOf(f.events, EventKernelStep); c != 3 {
		t.Fatalf("kernel events = %d: %v", c, f.events)
	}
	if !contains(f.events, EventApplied) {
		t.Fatal("an applied kernel gives the household its line")
	}
	if rep, _ := f.s.Store.LatestOSReport("hp", LayerGuest); rep != nil {
		t.Fatalf("a kernel report was stored as a guest report: %+v", rep)
	}
	if rep, _ := f.s.Store.LatestOSReport("hp", LayerKernel); rep == nil || rep.Outcome != "fell_back" {
		t.Fatalf("kernel report not stored on its layer: %+v", rep)
	}
}

// The operator's approval: every ring-0 box booted the same kernel healthily after a night stage — never before.
// COMPANION RED-PROOF (observed): drop the `night` check in KernelStatus → "approved without a night stage".
func TestKernel_ApproveNeedsEveryRing0BoxHealthyAfterANightStep(t *testing.T) {
	f := newFix(t)
	if _, err := f.s.ApproveKernel(); err == nil {
		t.Fatal("approved with nothing booted")
	}
	f.ingest(t, "hp", kreport("staged", "night", true))
	f.ingest(t, "hp", kreport("applied", "boot", true))
	f.ingest(t, "n100", kreport("staged", "signed", true)) // a by-day stage is not a night step
	f.ingest(t, "n100", kreport("applied", "boot", true))
	if _, err := f.s.ApproveKernel(); err == nil || !strings.Contains(err.Error(), "night") {
		t.Fatalf("approved without a night stage: %v", err)
	}
	f.ingest(t, "n100", kreport("staged", "night", true))
	f.ingest(t, "n100", kreport("applied", "boot", true))
	id, err := f.s.ApproveKernel()
	if err != nil || !strings.HasPrefix(id, "os-kernel-") {
		t.Fatalf("%q %v", id, err)
	}
	rel, _ := f.s.Store.LatestOSRelease(LayerKernel)
	if !strings.Contains(rel.PackagesJSON, `"proxmox-kernel-7.0"`) || !strings.Contains(rel.PackagesJSON, "proxmox-kernel-7.0.14-22-pve-signed") {
		t.Fatalf("release = %s", rel.PackagesJSON)
	}
	if contains(f.bumps, "cust1") {
		t.Fatal("an approved kernel set must nudge no ring-1 box (a signed job only)")
	}
	// a fell-back ring-0 box blocks the button
	g := newFix(t)
	g.ingest(t, "hp", kreport("staged", "night", true))
	g.ingest(t, "hp", kreport("fell_back", "boot", false))
	if st, _ := g.s.KernelStatus(); !strings.Contains(st.Waiting, "fell_back") {
		t.Fatalf("waiting = %q", st.Waiting)
	}
}

func contains(xs []string, x string) bool { return countOf(xs, x) > 0 }

func countOf(xs []string, x string) int {
	n := 0
	for _, v := range xs {
		if v == x {
			n++
		}
	}
	return n
}