hub v0.132.0: the System page (versions + OS updates with the ring/switch/approve buttons, R-852), the Hosts Proxmox/kernel column, the operator-approved Docker engine release (2 healthy ring-0 nights), the crash-guard events (R-851); evidence audits/os-docker-crash-2026-10-04

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS

hub/internal/osupdates/docker_test.go

@@ -0,0 +1,79 @@
+package osupdates
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+var engineSet = []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"},
+	{Name: "containerd.io", Version: "2.3.6-1~debian.13~trixie", Origin: "Docker"}}
+
+func (f *fix) dockerNight(t *testing.T, host string, healthy bool) {
+	t.Helper()
+	out := "nothing"
+	if !healthy {
+		out = "health_failed"
+	}
+	f.ingest(t, host, Report{Layer: LayerDocker, Trigger: "night", Mode: "apply", Outcome: out, Healthy: healthy,
+		Installed: append([]Package{pk("libc6", "x")}, engineSet...)})
+}
+
+// The Docker set is NEVER approved automatically (`11` §5.8: the operator approves it). Red-proof: add LayerDocker to
+// Layers (the auto-approved list) and this fails.
+func TestDocker_NeverAutoApproved(t *testing.T) {
+	f := newFix(t)
+	for i := 0; i < 3; i++ {
+		f.dockerNight(t, "hp", true)
+		f.dockerNight(t, "n100", true)
+		f.now = f.now.Add(25 * time.Hour)
+		f.s.Evaluate()
+	}
+	if rel, _ := f.s.Store.LatestOSRelease(LayerDocker); rel != nil {
+		t.Fatalf("a Docker set was auto-approved: %+v", rel)
+	}
+}
+
+// The operator's button works only after every ring-0 box ran the set 2 healthy nights; an unhealthy step blocks it.
+// The candidate is the six engine packages only. Red-proof: compare nights against 1 instead of DockerNights and the
+// one-night step approves.
+func TestDocker_ApproveNeedsTwoHealthyNightsOnEveryRing0Box(t *testing.T) {
+	f := newFix(t)
+	f.dockerNight(t, "hp", true)
+	f.dockerNight(t, "n100", true)
+	if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "1 of 2") {
+		t.Fatalf("approved after one night: %v", err)
+	}
+	f.now = f.now.Add(24 * time.Hour)
+	f.dockerNight(t, "hp", true)
+	if _, err := f.s.ApproveDocker(); err == nil {
+		t.Fatal("approved while n100 had only one night")
+	}
+	f.dockerNight(t, "n100", true)
+	id, err := f.s.ApproveDocker()
+	if err != nil || !strings.HasPrefix(id, "os-docker-") {
+		t.Fatalf("%q %v", id, err)
+	}
+	rel, _ := f.s.Store.LatestOSRelease(LayerDocker)
+	if rel.ApprovedBy != "operator" || !strings.Contains(rel.PackagesJSON, "docker-ce") || strings.Contains(rel.PackagesJSON, "libc6") {
+		t.Fatalf("release %+v", rel)
+	}
+	if len(f.bumps) != 0 {
+		t.Fatalf("a Docker approval must nudge no box (ring 1 takes it by a signed job): %v", f.bumps)
+	}
+	if b := f.s.DesiredBlock("cust1"); b.Release != nil || b.HostRelease != nil {
+		t.Fatalf("the Docker set leaked into the desired block: %+v", b)
+	}
+}
+
+func TestDocker_UnhealthyStepBlocksTheButton(t *testing.T) {
+	f := newFix(t)
+	f.dockerNight(t, "hp", true)
+	f.dockerNight(t, "n100", true)
+	f.now = f.now.Add(24 * time.Hour)
+	f.dockerNight(t, "hp", false)
+	f.dockerNight(t, "n100", true)
+	if _, err := f.s.ApproveDocker(); err == nil || !strings.Contains(err.Error(), "health_failed") {
+		t.Fatalf("an unhealthy Docker step did not block: %v", err)
+	}
+}

hub/internal/osupdates/service.go

@@ -33,13 +33,21 @@ import (
 
 // Layers.
 const (
-	LayerGuest = "guest"
-	LayerHost  = "host"
+	LayerGuest  = "guest"
+	LayerHost   = "host"
+	LayerDocker = "docker" // the guest's Docker engine set — slow lane, OPERATOR-approved only (`11` §5.8, hub v0.132.0)
 )
 
-// Layers lists every layer the hub approves.
+// Layers lists the layers the hub approves AUTOMATICALLY (the fast lane).
 var Layers = []string{LayerGuest, LayerHost}
 
+// AllLayers adds the Docker engine set (approved only by the operator's button, ApproveDocker).
+var AllLayers = []string{LayerGuest, LayerHost, LayerDocker}
+
+// dockerNames are the six packages of the Docker engine set (the agent wrapper's DOCKER_NAMES).
+var dockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
+	"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
+
 // hostSlowRE mirrors the wrapper's HOST_SLOW_RE (felhom-agent configs/felhom-os-apply): kernel, boot and firmware
 // packages are the host's slow lane and never enter a host release (the wrapper would refuse the whole plan, R14).
 // Pinned by TestCandidate_HostLeavesOutKernelBootFirmware.
@@ -91,6 +99,9 @@ type Report struct {
 	RebootScanned       bool            `json:"reboot_scanned,omitempty"` // agent ≥ 0.141.1: the pass looked (host: every pass)
 	Refused             json.RawMessage `json:"refused,omitempty"`
 	PassSeconds         float64         `json:"pass_seconds,omitempty"`
+	DockerEngine        string          `json:"docker_engine,omitempty"` // docker layer (agent v0.142.0)
+	Authority           string          `json:"authority,omitempty"`     // docker layer: ring0 | signed
+	Undo                bool            `json:"undo,omitempty"`          // docker layer: a signed undo
 }
 
 // PendingPkg is one update the sources offer.
@@ -145,6 +156,10 @@ type Service struct {
 	Emit           func(customerID, eventType, severity, message, details, source string)
 	ApproveAfter   time.Duration
 	NightsRequired int
+	// DockerNights is how many healthy ring-0 night runs of the Docker step an engine set needs before the operator's
+	// button may approve it (`11` §5.8: 2; 0 means 2; negative means none). OS_DOCKER_APPROVE_NIGHTS overrides it for
+	// a TEST only, logged.
+	DockerNights int
 	// Alarm thresholds (`11` §8.3; decided by CC unattended — operator may reverse). Zero = the default.
 	StaleAfter      time.Duration // 7 d
 	RebootAfter     time.Duration // 14 d
@@ -191,8 +206,9 @@ func (s *Service) event(customerID, typ, sev, msg string, details any) {
 }
 
 func layerOf(r Report) string {
-	if r.Layer == LayerHost {
-		return LayerHost
+	switch r.Layer {
+	case LayerHost, LayerDocker:
+		return r.Layer
 	}
 	return LayerGuest
 }
@@ -215,8 +231,11 @@ func (s *Service) Ingest(hostID string, r Report) error {
 	details := map[string]any{"host_id": hostID, "layer": layer, "run_id": r.RunID, "ring": r.Ring, "outcome": r.Outcome,
 		"upgraded": len(r.Upgraded), "release_id": r.ReleaseID, "health_reason": r.HealthReason}
 	where := "the box"
-	if layer == LayerHost {
+	switch layer {
+	case LayerHost:
 		where = "the box's base system"
+	case LayerDocker:
+		where = "the box's app engine (Docker " + r.DockerEngine + ")"
 	}
 	switch r.Outcome {
 	case "applied", "health_failed":
@@ -224,8 +243,11 @@ func (s *Service) Ingest(hostID string, r Report) error {
 			fmt.Sprintf("System security fixes installed on %s (%d package(s)).", where, len(r.Upgraded)), details)
 		if !r.Healthy || r.Outcome == "health_failed" {
 			undo := "last night's whole-guest backup is the undo (restore by hand, decision 81)"
-			if layer == LayerHost {
+			switch layer {
+			case LayerHost:
 				undo = "put a host package back by hand from the previous release's snapshot (runbook `os-updates-host-undo.md`)"
+			case LayerDocker:
+				undo = "sign an os_docker_step with undo for the previous engine set (runbook `os-updates-docker-undo.md`)"
 			}
 			s.event(h.CustomerID, EventHealthFailed, "error",
 				fmt.Sprintf("OS update (%s) on %s: NOT healthy after %d package(s) were installed (%s). Nothing was undone automatically; %s.",
@@ -254,7 +276,11 @@ func (s *Service) candidate(layer string, ring0 []string) (map[string]Package, e
 			return nil, err
 		}
 		for _, p := range r.Installed {
-			if p.Origin != "Debian" && p.Origin != "Debian-Security" {
+			if layer == LayerDocker {
+				if !dockerNames[p.Name] {
+					continue
+				}
+			} else if p.Origin != "Debian" && p.Origin != "Debian-Security" {
 				continue
 			}
 			if layer == LayerHost && hostSlowRE.MatchString(p.Name) {
@@ -322,6 +348,7 @@ type Status struct {
 	FirstSeen   time.Time
 	Packages    int
 	Waiting     string // why not approved yet ("" = approved or nothing to do)
+	Approved    string // the release id when this set is already approved
 }
 
 // Evaluate checks the approval rule of every layer and approves when it holds. Called every minute.
@@ -337,7 +364,12 @@ func (s *Service) Evaluate() ([]Status, error) {
 	return out, nil
 }
 
-func (s *Service) evaluateLayer(layer string) (Status, error) {
+func (s *Service) evaluateLayer(layer string) (Status, error) { return s.layerStatus(layer, true) }
+
+// LayerStatus is the read-only view of a fast-lane layer's candidate (the System page): it never approves.
+func (s *Service) LayerStatus(layer string) (Status, error) { return s.layerStatus(layer, false) }
+
+func (s *Service) layerStatus(layer string, approve bool) (Status, error) {
 	ring0, err := s.ring0Hosts()
 	if err != nil || len(ring0) == 0 {
 		return Status{Layer: layer, Waiting: "no ring-0 box"}, err
@@ -354,6 +386,7 @@ func (s *Service) evaluateLayer(layer string) (Status, error) {
 	}
 	st := Status{Layer: layer, Fingerprint: fp, FirstSeen: first, Packages: len(list)}
 	if rel, _ := s.Store.LatestOSRelease(layer); rel != nil && rel.Fingerprint == fp {
+		st.Approved = rel.ID
 		return st, nil // already approved
 	}
 	if age := s.now().Sub(first); age < s.ApproveAfter {
@@ -380,9 +413,53 @@ func (s *Service) evaluateLayer(layer string) (Status, error) {
 			return st, nil
 		}
 	}
+	if !approve {
+		st.Waiting = "ready — approves at the next minute's check"
+		return st, nil
+	}
 	return st, s.approve(layer, fp, list, "auto")
 }
 
+// ReleaseInfo is one layer's newest approved release, for the System page.
+type ReleaseInfo struct {
+	Layer      string
+	ID         string
+	ApprovedAt time.Time
+	ApprovedBy string
+	Packages   int
+}
+
+// Releases lists the newest release of every layer (guest, host, Docker); a layer with none is absent.
+func (s *Service) Releases() []ReleaseInfo {
+	var out []ReleaseInfo
+	for _, layer := range AllLayers {
+		rel, _ := s.Store.LatestOSRelease(layer)
+		if rel == nil {
+			continue
+		}
+		var list []Package
+		_ = json.Unmarshal([]byte(rel.PackagesJSON), &list)
+		out = append(out, ReleaseInfo{Layer: layer, ID: rel.ID, ApprovedAt: rel.ApprovedAt, ApprovedBy: rel.ApprovedBy, Packages: len(list)})
+	}
+	return out
+}
+
+// Candidates is the read-only candidate status of every layer (the Docker one by its own rule).
+func (s *Service) Candidates() []Status {
+	var out []Status
+	for _, layer := range Layers {
+		st, _ := s.LayerStatus(layer)
+		out = append(out, st)
+	}
+	d, _ := s.DockerStatus()
+	return append(out, d)
+}
+
+// Thresholds are the alarm numbers the System page colours by (the same values the alarms use).
+func (s *Service) Thresholds() (stale, reboot, notCovered time.Duration) {
+	return dflt(s.StaleAfter, 7*24*time.Hour), dflt(s.RebootAfter, 14*24*time.Hour), dflt(s.NotCoveredAfter, 14*24*time.Hour)
+}
+
 // ApproveNow approves every layer's current candidate at once (operator, urgent fix). Returns the release ids.
 func (s *Service) ApproveNow() (string, error) {
 	ring0, err := s.ring0Hosts()
@@ -418,6 +495,84 @@ func (s *Service) ApproveNow() (string, error) {
 	return strings.Join(ids, ","), nil
 }
 
+// DockerStatus is the Docker engine set ring 0 runs now and whether the operator's button may approve it: every ring-0
+// box has run it in DockerNights healthy night Docker steps since it was first seen, and none unhealthy.
+func (s *Service) DockerStatus() (Status, error) {
+	st := Status{Layer: LayerDocker}
+	ring0, err := s.ring0Hosts()
+	if err != nil || len(ring0) == 0 {
+		st.Waiting = "no ring-0 box"
+		return st, err
+	}
+	cand, err := s.candidate(LayerDocker, ring0)
+	if err != nil || len(cand) == 0 {
+		st.Waiting = "a ring-0 box has not reported a Docker step"
+		return st, err
+	}
+	fp, list := fingerprint(LayerDocker, cand)
+	pj, _ := json.Marshal(list)
+	first, err := s.Store.OSCandidateFirstSeen(fp, string(pj), s.now())
+	if err != nil {
+		return st, err
+	}
+	st.Fingerprint, st.FirstSeen, st.Packages = fp, first, len(list)
+	if rel, _ := s.Store.LatestOSRelease(LayerDocker); rel != nil && rel.Fingerprint == fp {
+		st.Approved, st.Waiting = rel.ID, "already approved"
+		return st, nil
+	}
+	need := s.DockerNights // 0 = the default 2; negative = none (a TEST override only, logged at start)
+	switch {
+	case need == 0:
+		need = 2
+	case need < 0:
+		need = 0
+	}
+	for _, h := range ring0 {
+		reps, err := s.Store.OSReportsSince(h, LayerDocker, first)
+		if err != nil {
+			return st, err
+		}
+		nights := 0
+		for _, r := range reps {
+			if !r.Healthy || r.Outcome == "failed" || r.Outcome == "refused" || r.Outcome == "health_failed" {
+				st.Waiting = fmt.Sprintf("%s reported a Docker step %s (healthy=%v) at %s", h, r.Outcome, r.Healthy, r.ReceivedAt.UTC().Format(time.RFC3339))
+				return st, nil
+			}
+			if r.Trigger == "night" {
+				nights++
+			}
+		}
+		if nights < need {
+			st.Waiting = fmt.Sprintf("%s has %d of %d healthy night Docker step(s) with this set", h, nights, need)
+			return st, nil
+		}
+	}
+	return st, nil
+}
+
+// ApproveDocker is the operator's button: it approves the Docker engine set ring 0 runs, only when DockerStatus allows.
+// A ring-1 box then takes it only through a signed operator job (`11` §5.8) — approval alone installs nothing.
+func (s *Service) ApproveDocker() (string, error) {
+	st, err := s.DockerStatus()
+	if err != nil {
+		return "", err
+	}
+	if st.Waiting != "" {
+		return "", fmt.Errorf("osupdates: the Docker set cannot be approved yet: %s", st.Waiting)
+	}
+	ring0, _ := s.ring0Hosts()
+	cand, err := s.candidate(LayerDocker, ring0)
+	if err != nil {
+		return "", err
+	}
+	fp, list := fingerprint(LayerDocker, cand)
+	if err := s.approve(LayerDocker, fp, list, "operator"); err != nil {
+		return "", err
+	}
+	rel, _ := s.Store.LatestOSRelease(LayerDocker)
+	return rel.ID, nil
+}
+
 func (s *Service) approve(layer, fp string, list []Package, by string) error {
 	at := s.now().UTC().Truncate(time.Second)
 	id := "os-" + layer + "-" + at.Format("20060102-150405")
@@ -428,7 +583,7 @@ func (s *Service) approve(layer, fp string, list []Package, by string) error {
 	s.logf("[WARN] osupdates: OS release %s (%s) APPROVED by %s (%d packages, fingerprint %s)", id, layer, by, len(list), fp)
 	s.event("", EventReleaseApprove, "info", fmt.Sprintf("OS release %s (%s) approved (%s, %d packages).", id, layer, by, len(list)),
 		map[string]any{"release_id": id, "layer": layer, "approved_by": by, "packages": len(list), "fingerprint": fp})
-	if s.Bump != nil {
+	if s.Bump != nil && layer != LayerDocker { // a Docker set reaches ring 1 only by a signed job, not the desired state
 		hosts, _ := s.Store.ListHosts()
 		for _, h := range hosts {
 			if st := s.Store.GetOSHostSettings(h.HostID); st.Ring == 1 && st.Enabled {
@@ -518,6 +673,7 @@ type FleetLine struct {
 	Tunnel  string
 	Guest   LayerLine
 	Host    LayerLine
+	Docker  LayerLine
 }
 
 func (s *Service) layerLine(hostID, layer string, ring int) LayerLine {
@@ -605,6 +761,7 @@ func (s *Service) Fleet() ([]FleetLine, error) {
 		}
 		l.Guest = s.layerLine(h.HostID, LayerGuest, st.Ring)
 		l.Host = s.layerLine(h.HostID, LayerHost, st.Ring)
+		l.Docker = s.layerLine(h.HostID, LayerDocker, st.Ring)
 		out = append(out, l)
 	}
 	return out, nil
@@ -617,7 +774,7 @@ func (s *Service) FleetJSON() (any, error) {
 		return nil, err
 	}
 	out := map[string]any{"boxes": lines}
-	for _, layer := range Layers {
+	for _, layer := range AllLayers {
 		if rel, _ := s.Store.LatestOSRelease(layer); rel != nil {
 			out["latest_"+layer+"_release"] = map[string]any{"id": rel.ID, "approved_at": rel.ApprovedAt, "approved_by": rel.ApprovedBy}
 		}