Docker engine slow lane (live-restore once by reload; ring-0 pending-docker under a root-owned ring-0 mark; ring 1 and undo only by a signed os_docker_step the wrapper re-verifies against a root-owned signers file; same-container-id health), the version report (facts mode -> host report system stanza, R-852), guest restart scan every pass (R-849), the crash guard (kernel.panic=10, the 3rd unclean stop in 60 min stays off, 24 h re-arm)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-04 16:14:36 +02:00
parent 2e2e8f56b8
commit b1746c25af
17 changed files with 1780 additions and 135 deletions
@@ -134,14 +134,14 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
 		LayerHost:  {Upgraded: []Package{{Name: "openssl", Version: "u3"}}},
 	}}
 	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
-	g, ho := l.Run(context.Background(), 9201, "night")
+	g, ho := run2(l, "night")
 	if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
 		t.Fatalf("guest %+v\nhost %+v", g, ho)
 	}
-	if calls(w) != "guest:apply,host:apply" {
-		t.Fatalf("calls = %s, want one apply per layer, guest first", calls(w))
+	if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply" {
+		t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore and the ring-0 docker step", calls(w))
 	}
-	for _, p := range w.plans {
+	for _, p := range w.plans[:2] {
 		if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
 			t.Fatalf("ring-0 plan = %v", p)
 		}
@@ -149,7 +149,7 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
 	if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
 		t.Fatalf("not covered = %v", g.NotCovered)
 	}
-	if len(h.reports) != 2 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost {
+	if len(h.reports) != 3 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker {
 		t.Fatalf("hub got %+v", h.reports)
 	}
 }
@@ -163,7 +163,7 @@ func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
 	gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}}
 	hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}}
 	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr})
-	g, ho := l.Run(context.Background(), 9201, "night")
+	g, ho := run2(l, "night")
 	if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" {
 		t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID)
 	}
@@ -182,7 +182,7 @@ func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
 func TestRing1_NoReleaseIsInventory(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend}
 	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
-	g, ho := l.Run(context.Background(), 9201, "night")
+	g, ho := run2(l, "night")
 	if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" {
 		t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
 	}
@@ -192,7 +192,7 @@ func TestRing1_NoReleaseIsInventory(t *testing.T) {
 func TestNoBlock_IsRing1Nothing(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend}
 	l, _ := newLeg(t, w, nil)
-	if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "nothing" || g.Ring != 1 {
+	if g, _ := run2(l, "night"); g.Outcome != "nothing" || g.Ring != 1 {
 		t.Fatalf("g=%+v calls=%s", g, calls(w))
 	}
 }
@@ -201,7 +201,7 @@ func TestNoBlock_IsRing1Nothing(t *testing.T) {
 func TestSwitchOff_ReportsOnly(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend}
 	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
-	g, ho := l.Run(context.Background(), 9201, "night")
+	g, ho := run2(l, "night")
 	if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 {
 		t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
 	}
@@ -212,8 +212,9 @@ func TestBYO_NoHostPlan(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
 	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
 	l.Appliance = false
-	_, ho := l.Run(context.Background(), 9201, "night")
-	if ho.Outcome != "" || calls(w) != "guest:apply" || len(h.reports) != 1 {
+	_, ho := run2(l, "night")
+	// the guest (and so its Docker engine) is ours on a BYO box too: only the HOST is the owner's
+	if ho.Outcome != "" || calls(w) != "guest:apply,guest:live-restore-on,docker:apply" || len(h.reports) != 2 {
 		t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w))
 	}
 }
@@ -223,7 +224,7 @@ func TestGuestFailure_SkipsTheHost(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
 		LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}}
 	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
-	g, ho := l.Run(context.Background(), 9201, "night")
+	g, ho := run2(l, "night")
 	if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" {
 		t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
 	}
@@ -236,7 +237,7 @@ func TestHealth_FailsAfterTheWait(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}},
 		healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}}
 	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
-	g, ho := l.Run(context.Background(), 9201, "night")
+	g, ho := run2(l, "night")
 	if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" {
 		t.Fatalf("g=%+v h=%+v", g, ho)
 	}
@@ -251,7 +252,7 @@ func TestHealth_RecoversInsideTheWait(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}},
 		healthSeq: map[string][]*Health{LayerGuest: {starting}}}
 	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
-	if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "applied" || !g.Healthy {
+	if g, _ := run2(l, "night"); g.Outcome != "applied" || !g.Healthy {
 		t.Fatalf("g = %+v", g)
 	}
 }
@@ -261,7 +262,7 @@ func TestHost_TunnelDownFailsTheHostStep(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}}
 	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
 	l.Tunnel = fakeTunnel{hub.TunnelNotRunning}
-	_, ho := l.Run(context.Background(), 9201, "night")
+	_, ho := run2(l, "night")
 	if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") {
 		t.Fatalf("host = %+v", ho)
 	}
@@ -328,12 +329,12 @@ func TestHostHealthVerdict(t *testing.T) {
 func TestOncePerNight(t *testing.T) {
 	w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
 	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
-	l.Run(context.Background(), 9201, "night")
+	run2(l, "night")
 	n := len(w.plans)
-	if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "skipped" || len(w.plans) != n {
+	if g, _ := run2(l, "night"); g.Outcome != "skipped" || len(w.plans) != n {
 		t.Fatalf("a second night run in the same night ran: %+v", g)
 	}
-	if g, _ := l.Run(context.Background(), 9201, "debug"); g.Outcome == "skipped" {
+	if g, _ := run2(l, "debug"); g.Outcome == "skipped" {
 		t.Fatal("the debug action must not be throttled")
 	}
 }
@@ -344,13 +345,16 @@ func TestWrapperSuite(t *testing.T) {
 	if err != nil {
 		t.Skip("python3 not available")
 	}
-	cmd := exec.Command(py, "-B", "../../configs/test_felhom_os_apply.py")
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("wrapper suite failed: %v\n%s", err, out)
-	}
-	if !strings.Contains(string(out), "OK") {
-		t.Fatalf("wrapper suite did not report OK:\n%s", out)
+	// the OS wrapper and (agent v0.142.0) the crash guard — both root programs in configs/ with their own suites
+	for _, suite := range []string{"../../configs/test_felhom_os_apply.py", "../../configs/test_felhom_crash_guard.py"} {
+		cmd := exec.Command(py, "-B", suite)
+		out, err := cmd.CombinedOutput()
+		if err != nil {
+			t.Fatalf("%s failed: %v\n%s", suite, err, out)
+		}
+		if !strings.Contains(string(out), "OK") {
+			t.Fatalf("%s did not report OK:\n%s", suite, out)
+		}
 	}
 }

@@ -362,8 +366,102 @@ func TestHostReport_CarriesRebootScanned(t *testing.T) {
 		LayerHost:  {RebootScanned: true, RebootNeeded: true, RestartNeeded: []string{"lxc-start"}},
 	}}
 	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
-	l.Run(context.Background(), 9201, "night")
-	if len(h.reports) != 2 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
+	run2(l, "night")
+	if len(h.reports) != 3 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
 		t.Fatalf("hub got %+v", h.reports)
 	}
 }
+
+// run2 is the guest + host reports of one pass (the tests written before the docker step).
+func run2(l *Leg, trigger string) (Report, Report) {
+	p := l.Run(context.Background(), 9201, trigger)
+	return p.Guest, p.Host
+}
+
+// ---- the Docker step (`11` §5.8, agent v0.142.0) ----
+
+// Ring 1 never takes an engine step in the night leg — only inside a signed operator job. Red-proof: drop the
+// `blk.Ring != 0` case in Run and the ring-1 pass makes a docker call.
+func TestDocker_Ring1NightLegNeverSteps(t *testing.T) {
+	w := &fakeWrapper{t: t}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
+	p := l.Run(context.Background(), 9201, "night")
+	if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") || strings.Contains(calls(w), "live-restore") {
+		t.Fatalf("ring 1 took a docker step: %s", calls(w))
+	}
+}
+
+// An unhealthy earlier step skips the docker step.
+func TestDocker_SkippedAfterAnUnhealthyStep(t *testing.T) {
+	w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}},
+		healthSeq: map[string][]*Health{}}
+	bad := guestOK()
+	bad.Controller = "unhealthy"
+	w.applyRep[LayerGuest] = WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}
+	w.healthSeq[LayerGuest] = []*Health{bad, bad, bad, bad, bad, bad, bad, bad}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	p := l.Run(context.Background(), 9201, "night")
+	if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") {
+		t.Fatalf("docker step ran after an unhealthy guest step: %s", calls(w))
+	}
+}
+
+// The docker plan is the slow lane, pending-docker for ring 0; the report carries only the engine set.
+func TestDocker_Ring0PlanAndReport(t *testing.T) {
+	w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
+		Upgraded:     []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}},
+		Installed:    []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"}, {Name: "libc6", Version: "u4", Origin: "Debian"}},
+		DockerEngine: "29.8.2", Authority: "ring0"}}}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	p := l.Run(context.Background(), 9201, "night")
+	dp := w.plans[len(w.plans)-1]
+	if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["select"] != "pending-docker" {
+		t.Fatalf("docker plan = %v", dp)
+	}
+	d := p.Docker
+	if d.Outcome != "applied" || !d.Healthy || d.DockerEngine != "29.8.2" || len(d.Installed) != 1 || d.Installed[0].Name != "docker-ce" {
+		t.Fatalf("docker report = %+v", d)
+	}
+}
+
+// THE docker health rule. Red-proof: drop the id comparison (or the engine check) in DockerHealthVerdict and a case fails.
+func TestDockerHealthVerdict(t *testing.T) {
+	before := guestOK()
+	before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
+		"app": {State: "running", Health: "healthy", ID: "b"}}
+	same := guestOK()
+	same.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
+		"app": {State: "running", Health: "healthy", ID: "b"}}
+	moved := guestOK()
+	moved.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
+		"app": {State: "running", Health: "healthy", ID: "c"}}
+	if ok, why := DockerHealthVerdict(before, same, "29.8.2", "29.8.2"); !ok {
+		t.Fatalf("same ids, right engine: %s", why)
+	}
+	if ok, _ := DockerHealthVerdict(before, moved, "29.8.2", "29.8.2"); ok {
+		t.Fatal("a changed container id passed — live-restore failed and the apps restarted")
+	}
+	if ok, _ := DockerHealthVerdict(before, same, "29.8.2", "29.7.2"); ok {
+		t.Fatal("the engine did not move and the step passed")
+	}
+	if EngineOf("5:29.8.2-1~debian.13~trixie") != "29.8.2" {
+		t.Fatalf("EngineOf = %q", EngineOf("5:29.8.2-1~debian.13~trixie"))
+	}
+}
+
+// A changed id after the step → health_failed (the consequence, not only the verdict).
+func TestDocker_ChangedIDIsHealthFailed(t *testing.T) {
+	before := guestOK()
+	before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}}
+	after := guestOK()
+	after.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "z"}}
+	w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
+		Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1"}}, DockerEngine: "29.8.2",
+		HealthBefore: before, HealthAfter: after}}, healthSeq: map[string][]*Health{}}
+	w.healthSeq[LayerDocker] = []*Health{after, after, after, after, after, after, after, after}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	p := l.Run(context.Background(), 9201, "night")
+	if p.Docker.Outcome != "health_failed" || !strings.Contains(p.Docker.HealthReason, "id changed") {
+		t.Fatalf("docker = %+v", p.Docker)
+	}
+}