package osupdates import ( "strings" "testing" "time" "gitea.dooplex.hu/admin/felhom-hub/internal/store" ) // ---------- the host layer (`11` §8 step 3) ---------- // The host release never carries a kernel, boot or firmware package (the host's slow lane, C3): the wrapper would // refuse the whole plan (R14). Red-proof: drop the hostSlowRE check in candidate() and this fails. func TestCandidate_HostLeavesOutKernelBootFirmware(t *testing.T) { f := newFix(t) set := []Package{pk("libssl3t64", "3.5.7-1~deb13u3"), pk("proxmox-kernel-6.14.11-4-pve-signed", "6.14.11-4"), pk("grub-efi-amd64", "2.12-9"), pk("firmware-realtek", "20250410-2"), pk("intel-microcode", "3.20250512.1"), pk("shim-signed", "1.46"), pk("pve-firmware", "3.16-3")} f.reportL(t, "hp", LayerHost, "night", true, set...) f.reportL(t, "n100", LayerHost, "night", true, set...) cand, err := f.s.candidate(LayerHost, []string{"hp", "n100"}) if err != nil { t.Fatal(err) } if len(cand) != 1 || cand["libssl3t64"].Version != "3.5.7-1~deb13u3" { t.Fatalf("host candidate = %+v, want only libssl3t64", cand) } // The guest layer has no such rule: a guest has no kernel of its own, nothing to leave out. f.reportL(t, "hp", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1")) f.reportL(t, "n100", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1")) if g, _ := f.s.candidate(LayerGuest, []string{"hp", "n100"}); len(g) != 1 { t.Fatalf("guest candidate = %+v", g) } } // Host and guest are SEPARATE approved sets: a guest night run does not count toward a host approval, and each layer // gets its own release id. Red-proof: count nights across layers (OSReportsSince without layer) and this fails. func TestLayers_SeparateSets(t *testing.T) { f := newFix(t) g := []Package{pk("libc6", "2.41-12+deb13u4")} h := []Package{pk("libssl3t64", "3.5.7-1~deb13u3")} for _, b := range []string{"hp", "n100"} { f.reportL(t, b, LayerGuest, "debug", true, g...) f.reportL(t, b, LayerHost, "debug", true, h...) } f.s.Evaluate() f.now = f.now.Add(25 * time.Hour) for _, b := range []string{"hp", "n100"} { f.reportL(t, b, LayerGuest, "night", true, g...) // a GUEST night only } sts, err := f.s.Evaluate() if err != nil || len(sts) != 2 { t.Fatalf("%v %+v", err, sts) } if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil || !strings.HasPrefix(gr.ID, "os-guest-") { t.Fatalf("guest release = %+v", gr) } if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil { t.Fatalf("host approved on a GUEST night: %+v", hr) } if !strings.Contains(sts[1].Waiting, "night run") || sts[1].Layer != LayerHost { t.Fatalf("host status = %+v", sts[1]) } for _, b := range []string{"hp", "n100"} { f.reportL(t, b, LayerHost, "night", true, h...) } f.s.Evaluate() hr, _ := f.s.Store.LatestOSRelease(LayerHost) if hr == nil || !strings.HasPrefix(hr.ID, "os-host-") { t.Fatalf("host release = %+v", hr) } b := f.s.DesiredBlock("cust1") if b.Release == nil || b.HostRelease == nil || b.HostRelease.Packages[0].Name != "libssl3t64" || b.Release.Packages[0].Name != "libc6" { t.Fatalf("ring-1 block = %+v", b) } } // An unhealthy HOST run blocks only the host approval. The undo text points to the host runbook, not the guest backup. func TestHostUnhealthy_BlocksHostOnly(t *testing.T) { f := newFix(t) for _, b := range []string{"hp", "n100"} { f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1")) f.reportL(t, b, LayerHost, "night", true, pk("libssl3t64", "1")) } f.s.Evaluate() f.now = f.now.Add(25 * time.Hour) var msgs []string f.s.Emit = func(_, typ, _, msg, _, _ string) { f.events = append(f.events, typ); msgs = append(msgs, msg) } for _, b := range []string{"hp", "n100"} { f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1")) } f.reportL(t, "hp", LayerHost, "night", false, pk("libssl3t64", "1")) f.reportL(t, "n100", LayerHost, "night", true, pk("libssl3t64", "1")) f.s.Evaluate() if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil { t.Fatal("an unhealthy HOST run blocked the GUEST approval") } if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil { t.Fatalf("host approved over an unhealthy host run: %+v", hr) } if !strings.Contains(strings.Join(msgs, "\n"), "os-updates-host-undo.md") { t.Fatalf("host health mail does not name the host undo runbook: %q", msgs) } } // ---------- the four alarms (`11` §8.3) ---------- func (f *fix) count(typ string) int { n := 0 for _, e := range f.events { if e == typ { n++ } } return n } func (f *fix) setAgent(t *testing.T, host, ver string) { t.Helper() if err := f.s.Store.SaveHostReport(host, "c-"+host, []byte(`{}`), store.HostReportDenorm{AgentVersion: ver}); err != nil { t.Fatal(err) } } func (f *fix) ingest(t *testing.T, host string, r Report) { t.Helper() if r.RunID == "" { r.RunID = host + f.now.String() } if err := f.s.Ingest(host, r); err != nil { t.Fatal(err) } } // Alarm 1: no completed OS leg for 7 days while the switch is ON, naming the likely reason; once a week at most; // cleared by a good leg. Red-proof: compare against a fixed 70 days (or drop the Enabled check) and a sub-step fails. func TestAlarm_StaleLeg(t *testing.T) { f := newFix(t) f.s.Now = func() time.Time { return f.now } f.now = time.Now().UTC() f.setAgent(t, "cust1", "0.141.0") f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true}) f.now = f.now.Add(6 * 24 * time.Hour) if sent, _ := f.s.Alarms(); f.count(EventStale) != 0 { t.Fatalf("alarm at 6 days: %v", sent) } f.now = f.now.Add(2 * 24 * time.Hour) f.s.Alarms() if f.count(EventStale) != 1 { t.Fatalf("no alarm at 8 days: %v", f.events) } f.now = f.now.Add(time.Hour) f.s.Alarms() if f.count(EventStale) != 1 { t.Fatal("re-alarmed within the week") } f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true}) f.s.Alarms() if raised := f.s.Store.OSAlarmRaised("stale:cust1"); !raised.IsZero() { t.Fatal("a good leg did not clear the alarm") } // The switch OFF: the leg is not expected, no alarm. _ = f.s.Store.SetOSEnabled("cust1", false) f.now = f.now.Add(30 * 24 * time.Hour) f.s.Alarms() if f.count(EventStale) != 1 { t.Fatal("alarmed with the switch OFF") } } // The stale alarm's reason: a failed leg is named; a box that runs an agent too old for the leg is not watched. func TestAlarm_StaleNamesTheReason(t *testing.T) { f := newFix(t) var msgs []string f.s.Emit = func(_, typ, _, msg, _, _ string) { if typ == EventStale { msgs = append(msgs, msg) } } // The leg reports are dated 8 days back; the host reports (real clock) are fresh — the box IS reporting. f.now = time.Now().UTC().Add(-8*24*time.Hour - time.Hour) f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true}) f.now = f.now.Add(time.Hour) f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "refused", HealthReason: "R6 dpkg lock"}) f.setAgent(t, "cust1", "0.141.0") f.setAgent(t, "hp", "0.139.0") // too old to run the leg: never watched f.now = time.Now().UTC() f.s.Alarms() if len(msgs) != 1 || !strings.Contains(msgs[0], "ended refused") || !strings.Contains(msgs[0], "Likely reason") { t.Fatalf("stale alarms = %q", msgs) } } // Alarm 2: reboot needed for more than 14 days (host layer), from the FIRST report that said so. // Red-proof: take `since` from the newest report instead of the oldest in the run and the 15-day step stays silent. func TestAlarm_RebootNeeded(t *testing.T) { f := newFix(t) up := []Package{pk("libc6", "2")} f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true}) for d := 1; d <= 13; d++ { f.now = f.now.Add(24 * time.Hour) f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true}) f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true}) } f.s.Alarms() if f.count(EventRebootNeeded) != 0 { t.Fatal("alarm at 13 days") } if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() || f.now.Sub(l.RebootNeededSince) != 13*24*time.Hour { t.Fatalf("reboot-needed since = %v", l.RebootNeededSince) } f.now = f.now.Add(2 * 24 * time.Hour) f.s.Alarms() if f.count(EventRebootNeeded) != 1 { t.Fatalf("no alarm at 15 days: %v", f.events) } // A guest-layer reboot flag never feeds the host alarm. if l := f.s.layerLine("hp", LayerGuest, 0); !l.RebootNeededSince.IsZero() { t.Fatal("guest line shows a reboot need") } } // Alarm 3: ring 0 approved nothing for 7 days while it has pending fast-lane updates. Pending SLOW-lane packages // (a new kernel) do not count. Red-proof: count all pending (not fastPending) and the kernel-only step alarms. func TestAlarm_Ring0Stalled(t *testing.T) { f := newFix(t) kernel := []PendingPkg{{Name: "proxmox-kernel-6.17", From: "", To: "6.17.1", Origin: []string{"Debian"}}} for _, b := range []string{"hp", "n100"} { f.ingest(t, b, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: kernel}) } f.now = f.now.Add(8 * 24 * time.Hour) f.s.Alarms() if f.count(EventRing0Stalled) != 0 { t.Fatal("a pending KERNEL counted as a stalled fast lane") } fast := []PendingPkg{{Name: "libssl3t64", From: "1", To: "2", Origin: []string{"Debian-Security"}}} f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "failed", Healthy: false, Pending: fast}) f.s.Alarms() if f.count(EventRing0Stalled) != 1 { t.Fatalf("no stall alarm: %v", f.events) } // A release clears it. _ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-host-x", Layer: LayerHost, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto", PackagesJSON: "[]"}) f.s.Alarms() if !f.s.Store.OSAlarmRaised("ring0stall:host").IsZero() { t.Fatal("a new release did not clear the stall alarm") } } // Alarm 4: a ring-1 box with fast-lane packages no release covers for more than 14 days. Ring 0 never alarms. // Red-proof: drop the ring==0 early return in notCoveredFast and the ring-0 step alarms. func TestAlarm_NotCovered(t *testing.T) { f := newFix(t) odd := []PendingPkg{{Name: "libsomething-hw", From: "1", To: "2", Origin: []string{"Debian"}}} _ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-guest-x", Layer: LayerGuest, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto", PackagesJSON: `[{"name":"libc6","version":"2","origin":"Debian"}]`}) for d := 0; d <= 15; d++ { f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd}) f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd}) if d == 13 { f.s.Alarms() if f.count(EventNotCovered) != 0 { t.Fatal("alarm at 13 days") } } f.now = f.now.Add(24 * time.Hour) } var msgs []string f.s.Emit = func(_, typ, _, msg, _, _ string) { f.events = append(f.events, typ) if typ == EventNotCovered { msgs = append(msgs, msg) } } f.s.Alarms() if len(msgs) != 1 || !strings.Contains(msgs[0], "cust1") || !strings.Contains(msgs[0], "libsomething-hw") { t.Fatalf("not-covered alarms = %q (a ring-0 box must never raise one)", msgs) } } // The fleet view: one line per box, both layers, the tunnel. func TestFleet_OneLinePerBoxBothLayers(t *testing.T) { f := newFix(t) if err := f.s.Store.SaveHostReport("hp", "c-hp", []byte(`{}`), store.HostReportDenorm{AgentVersion: "0.141.0", CloudflaredStatus: "running"}); err != nil { t.Fatal(err) } f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: []Package{pk("a", "1")}, PassSeconds: 8.5}) f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: []PendingPkg{{Name: "proxmox-kernel-6.17", Origin: []string{"Debian"}}}}) lines, err := f.s.Fleet() if err != nil || len(lines) != 3 { t.Fatalf("%v %d lines", err, len(lines)) } for _, l := range lines { if l.HostID != "hp" { continue } if l.Tunnel != "running" || l.Guest.LastOutcome != "applied" || l.Guest.WrapperPassSeconds != 8.5 || l.Host.LastOutcome != "nothing" || l.Host.Pending != 1 || l.Host.LastSuccessfulLeg.IsZero() { t.Fatalf("hp line = %+v", l) } return } t.Fatal("no hp line") } // A reboot clears "reboot needed": the agent scans the host on every pass (reboot_scanned) and a scanned pass that // finds nothing ends the run, so no alarm fires 14 days later. Red-proof: ignore RebootScanned in layerLine's predicate // and the cleared step still shows a date. func TestRebootNeeded_ClearedByAScannedPass(t *testing.T) { f := newFix(t) f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: []Package{pk("libc6", "2")}, RebootNeeded: true, RebootScanned: true}) f.now = f.now.Add(24 * time.Hour) f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, RebootNeeded: true, RebootScanned: true}) if l := f.s.layerLine("hp", LayerHost, 0); f.now.Sub(l.RebootNeededSince) != 24*time.Hour { t.Fatalf("since = %v", l.RebootNeededSince) } f.now = f.now.Add(24 * time.Hour) // the operator rebooted f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, RebootScanned: true}) if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.IsZero() { t.Fatalf("a scanned pass after the reboot did not clear it: %v", l.RebootNeededSince) } f.now = f.now.Add(20 * 24 * time.Hour) f.s.Alarms() if f.count(EventRebootNeeded) != 0 { t.Fatal("reboot alarm after the reboot") } }