hub v0.140.0: a failed operator mail is retried (1, 5, 15 min); a Docker set needs a passing memory-kill check on every ring-0 box (decision 157)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

hub/internal/osupdates/docker_test.go

@@ -1,6 +1,7 @@
 package osupdates
 
 import (
+	"encoding/json"
 	"strings"
 	"testing"
 	"time"
@@ -10,13 +11,23 @@ var engineSet = []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~tri
 	{Name: "containerd.io", Version: "2.3.6-1~debian.13~trixie", Origin: "Docker"}}
 
 func (f *fix) dockerNight(t *testing.T, host string, healthy bool) {
+	t.Helper()
+	f.dockerNightOOM(t, host, healthy, `{"result":"pass","oom_killed":true,"oom_event":true,"exit_code":137,"image":"felhom-controller","detail":""}`)
+}
+
+// dockerNightOOM is dockerNight with the step's memory-kill check as given ("" = an agent that reports none).
+func (f *fix) dockerNightOOM(t *testing.T, host string, healthy bool, oom string) {
 	t.Helper()
 	out := "nothing"
 	if !healthy {
 		out = "health_failed"
 	}
-	f.ingest(t, host, Report{Layer: LayerDocker, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy,
-		Installed: append([]Package{pk("libc6", "x")}, engineSet...)})
+	r := Report{Layer: LayerDocker, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy,
+		Installed: append([]Package{pk("libc6", "x")}, engineSet...)}
+	if oom != "" {
+		r.OOMCheck = json.RawMessage(oom)
+	}
+	f.ingest(t, host, r)
 }
 
 // The Docker set is NEVER approved automatically (`11` §5.8: the operator approves it). Red-proof: add LayerDocker to
@@ -77,3 +88,67 @@ func TestDocker_UnhealthyStepBlocksTheButton(t *testing.T) {
 		t.Fatalf("an unhealthy Docker step did not block: %v", err)
 	}
 }
+
+// Decision 157 (R-528): the Docker set is approved only when every ring-0 box's step showed the engine reports a
+// memory kill. RED-PROOF (REPORT): make oomCheckWaiting return "" — the three blocking tests below approve.
+func TestDockerApproval_AFailedMemoryKillCheckBlocks(t *testing.T) {
+	f := newFix(t)
+	fail := `{"result":"fail","oom_killed":false,"oom_event":true,"exit_code":137,"image":"felhom-controller","detail":"OOMKilled=false"}`
+	for i := 0; i < 2; i++ {
+		f.dockerNight(t, "hp", true)
+		f.dockerNightOOM(t, "n100", true, fail)
+		if i == 0 {
+			f.s.Evaluate() // stamps the set\'s first-seen at the first night, as the tick does
+		}
+		f.now = f.now.Add(24 * time.Hour)
+	}
+	if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "memory-kill check did not pass") {
+		t.Fatalf("a set whose engine missed a memory kill was approvable: %v", err)
+	}
+}
+
+func TestDockerApproval_AMissingMemoryKillCheckBlocks(t *testing.T) {
+	f := newFix(t)
+	for i := 0; i < 2; i++ {
+		f.dockerNight(t, "hp", true)
+		f.dockerNightOOM(t, "n100", true, "") // an older agent: no check at all
+		if i == 0 {
+			f.s.Evaluate() // stamps the set\'s first-seen at the first night, as the tick does
+		}
+		f.now = f.now.Add(24 * time.Hour)
+	}
+	if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "has not reported the Docker step's memory-kill check") {
+		t.Fatalf("a set with no memory-kill check was approvable: %v", err)
+	}
+}
+
+func TestDockerApproval_AnErroredMemoryKillCheckBlocks(t *testing.T) {
+	f := newFix(t)
+	errd := `{"result":"error","oom_killed":false,"oom_event":false,"exit_code":null,"image":null,"detail":"the controller's image could not be read"}`
+	for i := 0; i < 2; i++ {
+		f.dockerNight(t, "hp", true)
+		f.dockerNightOOM(t, "n100", true, errd)
+		if i == 0 {
+			f.s.Evaluate() // stamps the set\'s first-seen at the first night, as the tick does
+		}
+		f.now = f.now.Add(24 * time.Hour)
+	}
+	if _, err := f.s.ApproveDocker(); err == nil {
+		t.Fatal("a set whose memory-kill check errored was approvable")
+	}
+}
+
+func TestDockerApproval_APassingCheckOnEveryBoxAllows(t *testing.T) {
+	f := newFix(t)
+	for i := 0; i < 2; i++ {
+		f.dockerNight(t, "hp", true)
+		f.dockerNight(t, "n100", true)
+		if i == 0 {
+			f.s.Evaluate() // stamps the set\'s first-seen at the first night, as the tick does
+		}
+		f.now = f.now.Add(24 * time.Hour)
+	}
+	if _, err := f.s.ApproveDocker(); err != nil {
+		t.Fatalf("passing checks on every ring-0 box should allow the approval: %v", err)
+	}
+}