admin
fd1f98547e
gates / gates (push) Successful in 30s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
331 lines
14 KiB
Go
331 lines
14 KiB
Go
package osupdates
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.dooplex.hu/admin/felhom-hub/internal/store"
|
|
)
|
|
|
|
// ---------- the host layer (`11` §8 step 3) ----------
|
|
|
|
// The host release never carries a kernel, boot or firmware package (the host's slow lane, C3): the wrapper would
|
|
// refuse the whole plan (R14). Red-proof: drop the hostSlowRE check in candidate() and this fails.
|
|
func TestCandidate_HostLeavesOutKernelBootFirmware(t *testing.T) {
|
|
f := newFix(t)
|
|
set := []Package{pk("libssl3t64", "3.5.7-1~deb13u3"), pk("proxmox-kernel-6.14.11-4-pve-signed", "6.14.11-4"),
|
|
pk("grub-efi-amd64", "2.12-9"), pk("firmware-realtek", "20250410-2"), pk("intel-microcode", "3.20250512.1"),
|
|
pk("shim-signed", "1.46"), pk("pve-firmware", "3.16-3")}
|
|
f.reportL(t, "hp", LayerHost, "night", true, set...)
|
|
f.reportL(t, "n100", LayerHost, "night", true, set...)
|
|
cand, err := f.s.candidate(LayerHost, []string{"hp", "n100"})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(cand) != 1 || cand["libssl3t64"].Version != "3.5.7-1~deb13u3" {
|
|
t.Fatalf("host candidate = %+v, want only libssl3t64", cand)
|
|
}
|
|
// The guest layer has no such rule: a guest has no kernel of its own, nothing to leave out.
|
|
f.reportL(t, "hp", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1"))
|
|
f.reportL(t, "n100", LayerGuest, "night", true, pk("firmware-misc-nonfree", "1"))
|
|
if g, _ := f.s.candidate(LayerGuest, []string{"hp", "n100"}); len(g) != 1 {
|
|
t.Fatalf("guest candidate = %+v", g)
|
|
}
|
|
}
|
|
|
|
// Host and guest are SEPARATE approved sets: a guest night run does not count toward a host approval, and each layer
|
|
// gets its own release id. Red-proof: count nights across layers (OSReportsSince without layer) and this fails.
|
|
func TestLayers_SeparateSets(t *testing.T) {
|
|
f := newFix(t)
|
|
g := []Package{pk("libc6", "2.41-12+deb13u4")}
|
|
h := []Package{pk("libssl3t64", "3.5.7-1~deb13u3")}
|
|
for _, b := range []string{"hp", "n100"} {
|
|
f.reportL(t, b, LayerGuest, "debug", true, g...)
|
|
f.reportL(t, b, LayerHost, "debug", true, h...)
|
|
}
|
|
f.s.Evaluate()
|
|
f.now = f.now.Add(25 * time.Hour)
|
|
for _, b := range []string{"hp", "n100"} {
|
|
f.reportL(t, b, LayerGuest, "night", true, g...) // a GUEST night only
|
|
}
|
|
sts, err := f.s.Evaluate()
|
|
if err != nil || len(sts) != 2 {
|
|
t.Fatalf("%v %+v", err, sts)
|
|
}
|
|
if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil || !strings.HasPrefix(gr.ID, "os-guest-") {
|
|
t.Fatalf("guest release = %+v", gr)
|
|
}
|
|
if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil {
|
|
t.Fatalf("host approved on a GUEST night: %+v", hr)
|
|
}
|
|
if !strings.Contains(sts[1].Waiting, "night run") || sts[1].Layer != LayerHost {
|
|
t.Fatalf("host status = %+v", sts[1])
|
|
}
|
|
for _, b := range []string{"hp", "n100"} {
|
|
f.reportL(t, b, LayerHost, "night", true, h...)
|
|
}
|
|
f.s.Evaluate()
|
|
hr, _ := f.s.Store.LatestOSRelease(LayerHost)
|
|
if hr == nil || !strings.HasPrefix(hr.ID, "os-host-") {
|
|
t.Fatalf("host release = %+v", hr)
|
|
}
|
|
b := f.s.DesiredBlock("cust1")
|
|
if b.Release == nil || b.HostRelease == nil || b.HostRelease.Packages[0].Name != "libssl3t64" || b.Release.Packages[0].Name != "libc6" {
|
|
t.Fatalf("ring-1 block = %+v", b)
|
|
}
|
|
}
|
|
|
|
// An unhealthy HOST run blocks only the host approval. The undo text points to the host runbook, not the guest backup.
|
|
func TestHostUnhealthy_BlocksHostOnly(t *testing.T) {
|
|
f := newFix(t)
|
|
for _, b := range []string{"hp", "n100"} {
|
|
f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1"))
|
|
f.reportL(t, b, LayerHost, "night", true, pk("libssl3t64", "1"))
|
|
}
|
|
f.s.Evaluate()
|
|
f.now = f.now.Add(25 * time.Hour)
|
|
var msgs []string
|
|
f.s.Emit = func(_, typ, _, msg, _, _ string) { f.events = append(f.events, typ); msgs = append(msgs, msg) }
|
|
for _, b := range []string{"hp", "n100"} {
|
|
f.reportL(t, b, LayerGuest, "night", true, pk("libc6", "1"))
|
|
}
|
|
f.reportL(t, "hp", LayerHost, "night", false, pk("libssl3t64", "1"))
|
|
f.reportL(t, "n100", LayerHost, "night", true, pk("libssl3t64", "1"))
|
|
f.s.Evaluate()
|
|
if gr, _ := f.s.Store.LatestOSRelease(LayerGuest); gr == nil {
|
|
t.Fatal("an unhealthy HOST run blocked the GUEST approval")
|
|
}
|
|
if hr, _ := f.s.Store.LatestOSRelease(LayerHost); hr != nil {
|
|
t.Fatalf("host approved over an unhealthy host run: %+v", hr)
|
|
}
|
|
if !strings.Contains(strings.Join(msgs, "\n"), "os-updates-host-undo.md") {
|
|
t.Fatalf("host health mail does not name the host undo runbook: %q", msgs)
|
|
}
|
|
}
|
|
|
|
// ---------- the four alarms (`11` §8.3) ----------
|
|
|
|
func (f *fix) count(typ string) int {
|
|
n := 0
|
|
for _, e := range f.events {
|
|
if e == typ {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func (f *fix) setAgent(t *testing.T, host, ver string) {
|
|
t.Helper()
|
|
if err := f.s.Store.SaveHostReport(host, "c-"+host, []byte(`{}`), store.HostReportDenorm{AgentVersion: ver}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func (f *fix) ingest(t *testing.T, host string, r Report) {
|
|
t.Helper()
|
|
if r.RunID == "" {
|
|
r.RunID = host + f.now.String()
|
|
}
|
|
if err := f.s.Ingest(host, r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// Alarm 1: no completed OS leg for 7 days while the switch is ON, naming the likely reason; once a week at most;
|
|
// cleared by a good leg. Red-proof: compare against a fixed 70 days (or drop the Enabled check) and a sub-step fails.
|
|
func TestAlarm_StaleLeg(t *testing.T) {
|
|
f := newFix(t)
|
|
f.s.Now = func() time.Time { return f.now }
|
|
f.now = time.Now().UTC()
|
|
f.setAgent(t, "cust1", "0.141.0")
|
|
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true})
|
|
f.now = f.now.Add(6 * 24 * time.Hour)
|
|
if sent, _ := f.s.Alarms(); f.count(EventStale) != 0 {
|
|
t.Fatalf("alarm at 6 days: %v", sent)
|
|
}
|
|
f.now = f.now.Add(2 * 24 * time.Hour)
|
|
f.s.Alarms()
|
|
if f.count(EventStale) != 1 {
|
|
t.Fatalf("no alarm at 8 days: %v", f.events)
|
|
}
|
|
f.now = f.now.Add(time.Hour)
|
|
f.s.Alarms()
|
|
if f.count(EventStale) != 1 {
|
|
t.Fatal("re-alarmed within the week")
|
|
}
|
|
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true})
|
|
f.s.Alarms()
|
|
if raised := f.s.Store.OSAlarmRaised("stale:cust1"); !raised.IsZero() {
|
|
t.Fatal("a good leg did not clear the alarm")
|
|
}
|
|
// The switch OFF: the leg is not expected, no alarm.
|
|
_ = f.s.Store.SetOSEnabled("cust1", false)
|
|
f.now = f.now.Add(30 * 24 * time.Hour)
|
|
f.s.Alarms()
|
|
if f.count(EventStale) != 1 {
|
|
t.Fatal("alarmed with the switch OFF")
|
|
}
|
|
}
|
|
|
|
// The stale alarm's reason: a failed leg is named; a box that runs an agent too old for the leg is not watched.
|
|
func TestAlarm_StaleNamesTheReason(t *testing.T) {
|
|
f := newFix(t)
|
|
var msgs []string
|
|
f.s.Emit = func(_, typ, _, msg, _, _ string) {
|
|
if typ == EventStale {
|
|
msgs = append(msgs, msg)
|
|
}
|
|
}
|
|
// The leg reports are dated 8 days back; the host reports (real clock) are fresh — the box IS reporting.
|
|
f.now = time.Now().UTC().Add(-8*24*time.Hour - time.Hour)
|
|
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true})
|
|
f.now = f.now.Add(time.Hour)
|
|
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "refused", HealthReason: "R6 dpkg lock"})
|
|
f.setAgent(t, "cust1", "0.141.0")
|
|
f.setAgent(t, "hp", "0.139.0") // too old to run the leg: never watched
|
|
f.now = time.Now().UTC()
|
|
f.s.Alarms()
|
|
if len(msgs) != 1 || !strings.Contains(msgs[0], "ended refused") || !strings.Contains(msgs[0], "Likely reason") {
|
|
t.Fatalf("stale alarms = %q", msgs)
|
|
}
|
|
}
|
|
|
|
// Alarm 2: reboot needed for more than 14 days (host layer), from the FIRST report that said so.
|
|
// Red-proof: take `since` from the newest report instead of the oldest in the run and the 15-day step stays silent.
|
|
func TestAlarm_RebootNeeded(t *testing.T) {
|
|
f := newFix(t)
|
|
up := []Package{pk("libc6", "2")}
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true})
|
|
for d := 1; d <= 13; d++ {
|
|
f.now = f.now.Add(24 * time.Hour)
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true})
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: up, RebootNeeded: true})
|
|
}
|
|
f.s.Alarms()
|
|
if f.count(EventRebootNeeded) != 0 {
|
|
t.Fatal("alarm at 13 days")
|
|
}
|
|
if l := f.s.layerLine("hp", LayerHost, 0); l.RebootNeededSince.IsZero() || f.now.Sub(l.RebootNeededSince) != 13*24*time.Hour {
|
|
t.Fatalf("reboot-needed since = %v", l.RebootNeededSince)
|
|
}
|
|
f.now = f.now.Add(2 * 24 * time.Hour)
|
|
f.s.Alarms()
|
|
if f.count(EventRebootNeeded) != 1 {
|
|
t.Fatalf("no alarm at 15 days: %v", f.events)
|
|
}
|
|
// A guest-layer reboot flag never feeds the host alarm.
|
|
if l := f.s.layerLine("hp", LayerGuest, 0); !l.RebootNeededSince.IsZero() {
|
|
t.Fatal("guest line shows a reboot need")
|
|
}
|
|
}
|
|
|
|
// Alarm 3: ring 0 approved nothing for 7 days while it has pending fast-lane updates. Pending SLOW-lane packages
|
|
// (a new kernel) do not count. Red-proof: count all pending (not fastPending) and the kernel-only step alarms.
|
|
func TestAlarm_Ring0Stalled(t *testing.T) {
|
|
f := newFix(t)
|
|
kernel := []PendingPkg{{Name: "proxmox-kernel-6.17", From: "", To: "6.17.1", Origin: []string{"Debian"}}}
|
|
for _, b := range []string{"hp", "n100"} {
|
|
f.ingest(t, b, Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: kernel})
|
|
}
|
|
f.now = f.now.Add(8 * 24 * time.Hour)
|
|
f.s.Alarms()
|
|
if f.count(EventRing0Stalled) != 0 {
|
|
t.Fatal("a pending KERNEL counted as a stalled fast lane")
|
|
}
|
|
fast := []PendingPkg{{Name: "libssl3t64", From: "1", To: "2", Origin: []string{"Debian-Security"}}}
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "failed", Healthy: false, Pending: fast})
|
|
f.s.Alarms()
|
|
if f.count(EventRing0Stalled) != 1 {
|
|
t.Fatalf("no stall alarm: %v", f.events)
|
|
}
|
|
// A release clears it.
|
|
_ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-host-x", Layer: LayerHost, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto", PackagesJSON: "[]"})
|
|
f.s.Alarms()
|
|
if !f.s.Store.OSAlarmRaised("ring0stall:host").IsZero() {
|
|
t.Fatal("a new release did not clear the stall alarm")
|
|
}
|
|
}
|
|
|
|
// Alarm 4: a ring-1 box with fast-lane packages no release covers for more than 14 days. Ring 0 never alarms.
|
|
// Red-proof: drop the ring==0 early return in notCoveredFast and the ring-0 step alarms.
|
|
func TestAlarm_NotCovered(t *testing.T) {
|
|
f := newFix(t)
|
|
odd := []PendingPkg{{Name: "libsomething-hw", From: "1", To: "2", Origin: []string{"Debian"}}}
|
|
_ = f.s.Store.SaveOSRelease(store.OSRelease{ID: "os-guest-x", Layer: LayerGuest, Fingerprint: "x", ApprovedAt: f.now, ApprovedBy: "auto",
|
|
PackagesJSON: `[{"name":"libc6","version":"2","origin":"Debian"}]`})
|
|
for d := 0; d <= 15; d++ {
|
|
f.ingest(t, "cust1", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd})
|
|
f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, Pending: odd})
|
|
if d == 13 {
|
|
f.s.Alarms()
|
|
if f.count(EventNotCovered) != 0 {
|
|
t.Fatal("alarm at 13 days")
|
|
}
|
|
}
|
|
f.now = f.now.Add(24 * time.Hour)
|
|
}
|
|
var msgs []string
|
|
f.s.Emit = func(_, typ, _, msg, _, _ string) {
|
|
f.events = append(f.events, typ)
|
|
if typ == EventNotCovered {
|
|
msgs = append(msgs, msg)
|
|
}
|
|
}
|
|
f.s.Alarms()
|
|
if len(msgs) != 1 || !strings.Contains(msgs[0], "cust1") || !strings.Contains(msgs[0], "libsomething-hw") {
|
|
t.Fatalf("not-covered alarms = %q (a ring-0 box must never raise one)", msgs)
|
|
}
|
|
}
|
|
|
|
// The fleet view: one line per box, both layers, the tunnel.
|
|
func TestFleet_OneLinePerBoxBothLayers(t *testing.T) {
|
|
f := newFix(t)
|
|
if err := f.s.Store.SaveHostReport("hp", "c-hp", []byte(`{}`), store.HostReportDenorm{AgentVersion: "0.141.0", CloudflaredStatus: "running"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f.ingest(t, "hp", Report{Layer: LayerGuest, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true, Upgraded: []Package{pk("a", "1")}, PassSeconds: 8.5})
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true,
|
|
Pending: []PendingPkg{{Name: "proxmox-kernel-6.17", Origin: []string{"Debian"}}}})
|
|
lines, err := f.s.Fleet()
|
|
if err != nil || len(lines) != 3 {
|
|
t.Fatalf("%v %d lines", err, len(lines))
|
|
}
|
|
for _, l := range lines {
|
|
if l.HostID != "hp" {
|
|
continue
|
|
}
|
|
if l.Tunnel != "running" || l.Guest.LastOutcome != "applied" || l.Guest.WrapperPassSeconds != 8.5 ||
|
|
l.Host.LastOutcome != "nothing" || l.Host.Pending != 1 || l.Host.LastSuccessfulLeg.IsZero() {
|
|
t.Fatalf("hp line = %+v", l)
|
|
}
|
|
return
|
|
}
|
|
t.Fatal("no hp line")
|
|
}
|
|
|
|
// A reboot clears "reboot needed": the agent scans the host on every pass (reboot_scanned) and a scanned pass that
|
|
// finds nothing ends the run, so no alarm fires 14 days later. Red-proof: ignore RebootScanned in layerLine's predicate
|
|
// and the cleared step still shows a date.
|
|
func TestRebootNeeded_ClearedByAScannedPass(t *testing.T) {
|
|
f := newFix(t)
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "applied", Healthy: true,
|
|
Upgraded: []Package{pk("libc6", "2")}, RebootNeeded: true, RebootScanned: true})
|
|
f.now = f.now.Add(24 * time.Hour)
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, RebootNeeded: true, RebootScanned: true})
|
|
if l := f.s.layerLine("hp", LayerHost, 0); f.now.Sub(l.RebootNeededSince) != 24*time.Hour {
|
|
t.Fatalf("since = %v", l.RebootNeededSince)
|
|
}
|
|
f.now = f.now.Add(24 * time.Hour) // the operator rebooted
|
|
f.ingest(t, "hp", Report{Layer: LayerHost, Trigger: "night", Mode: "apply", Outcome: "nothing", Healthy: true, RebootScanned: true})
|
|
if l := f.s.layerLine("hp", LayerHost, 0); !l.RebootNeededSince.IsZero() {
|
|
t.Fatalf("a scanned pass after the reboot did not clear it: %v", l.RebootNeededSince)
|
|
}
|
|
f.now = f.now.Add(20 * 24 * time.Hour)
|
|
f.s.Alarms()
|
|
if f.count(EventRebootNeeded) != 0 {
|
|
t.Fatal("reboot alarm after the reboot")
|
|
}
|
|
}
|