Docker engine slow lane (live-restore once by reload; ring-0 pending-docker under a root-owned ring-0 mark; ring 1 and undo only by a signed os_docker_step the wrapper re-verifies against a root-owned signers file; same-container-id health), the version report (facts mode -> host report system stanza, R-852), guest restart scan every pass (R-849), the crash guard (kernel.panic=10, the 3rd unclean stop in 60 min stays off, 24 h re-arm)
gates / gates (push) Successful in 19s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 16:14:36 +02:00
parent 2e2e8f56b8
commit b1746c25af
17 changed files with 1780 additions and 135 deletions
+204 -16
View File
@@ -1,5 +1,7 @@
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3): the customer GUEST's Debian fast
// lane (agent v0.140.0) and, after it in the same pass, the HOST's (agent v0.141.0).
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3, §5.8): the customer GUEST's Debian
// fast lane (agent v0.140.0), after it in the same pass the HOST's (agent v0.141.0), and then — ring 0 only — the
// guest's DOCKER engine set, the slow lane (agent v0.142.0; a ring-1 box takes a Docker step only inside a signed
// operator job, DockerStepExecutor). It also reads the box's versions for the hub's System page (Facts, R-852).
//
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
@@ -35,10 +37,15 @@ const DefaultPlanDir = "/var/lib/felhom-agent/os"
// Layers.
const (
LayerGuest = "guest"
LayerHost = "host"
LayerGuest = "guest"
LayerHost = "host"
LayerDocker = "docker" // the guest's Docker engine set — slow lane (`11` §5.8)
)
// DockerNames are the six packages of the Docker engine set (the wrapper's DOCKER_NAMES).
var DockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
// Package is one name=version with its origin.
type Package struct {
Name string `json:"name"`
@@ -58,6 +65,7 @@ type Pending struct {
type Container struct {
State string `json:"state"`
Health string `json:"health"` // healthy | unhealthy | starting | none
ID string `json:"id,omitempty"`
}
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
@@ -89,6 +97,11 @@ type WrapperReport struct {
HealthAfter *Health `json:"health_after"`
Health *Health `json:"health"`
PassSeconds float64 `json:"pass_seconds"`
DockerEngine string `json:"docker_engine"`
Authority string `json:"authority"`
Undo bool `json:"undo"`
LiveRestore json.RawMessage `json:"live_restore"`
Facts json.RawMessage `json:"facts"`
}
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
@@ -116,6 +129,9 @@ type Report struct {
RebootScanned bool `json:"reboot_scanned,omitempty"` // the pass looked (host: every pass) — a false RebootNeeded then means "not needed"
Refused json.RawMessage `json:"refused,omitempty"`
PassSeconds float64 `json:"pass_seconds,omitempty"`
DockerEngine string `json:"docker_engine,omitempty"` // docker layer: the engine after the step
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo (downgrade)
}
// Reporter posts a report to the hub (*hub.Client).
@@ -284,6 +300,49 @@ func HostHealthVerdict(before, after *Health, tunnel string) (bool, string) {
return true, ""
}
// EngineOf is the engine version `docker version` prints for a docker-ce package version: "5:29.8.2-1~debian.13~trixie"
// → "29.8.2" (no epoch, no Debian revision).
func EngineOf(pkgVersion string) string {
v := pkgVersion
if i := strings.Index(v, ":"); i >= 0 {
v = v[i+1:]
}
if i := strings.Index(v, "-"); i >= 0 {
v = v[:i]
}
return v
}
// DockerHealthVerdict is THE Docker-step health rule (`11` §5.8; pinned by TestDockerHealthVerdict): the guest rule,
// plus every container running at the start still runs as the SAME container (same id — a changed id means the
// household's apps restarted, which `live-restore` exists to prevent), plus the engine now reports the version the step
// installed (wantEngine "" = no engine change expected).
func DockerHealthVerdict(before, after *Health, wantEngine, gotEngine string) (bool, string) {
if ok, why := HealthVerdict(before, after); !ok {
return false, why
}
if before != nil {
names := make([]string, 0, len(before.Containers))
for n := range before.Containers {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
b := before.Containers[n]
if b.State != "running" || b.ID == "" {
continue
}
if a := after.Containers[n]; a.ID != b.ID {
return false, n + " is a new container (id changed) — the engine step restarted it"
}
}
}
if wantEngine != "" && gotEngine != wantEngine {
return false, "the engine is " + gotEngine + ", not " + wantEngine
}
return true, ""
}
// call writes the plan and runs the wrapper once.
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
dir := l.PlanDir
@@ -320,9 +379,85 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
}
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer. Returns both
// reports (host empty when skipped). trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
// Pass is one leg's reports; an empty Layer means the step did not run.
type Pass struct {
Guest, Host, Docker Report
}
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
// only, after good earlier steps) the Docker engine set. trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
g, h := l.runFast(ctx, vmid, trigger)
p := Pass{Guest: g, Host: h}
if g.Outcome == "skipped" {
return p
}
blk := l.Block()
okStep := func(r Report) bool {
return (r.Outcome == "applied" || r.Outcome == "nothing" || r.Outcome == "inventory") && r.Healthy
}
lg := l.log().With("run", g.RunID, "vmid", vmid, "trigger", trigger)
switch {
case blk.Ring != 0 || !blk.Enabled:
lg.Info("osupdate: docker step skipped — ring 1 takes an engine set only inside a signed operator job (`11` §5.8)", "ring", blk.Ring, "enabled", blk.Enabled)
case !okStep(g) || (h.Layer != "" && !okStep(h)):
lg.Warn("osupdate: docker step skipped — an earlier step did not end healthy")
default:
if err := l.EnsureLiveRestore(ctx, g.RunID, vmid); err != nil {
p.Docker = l.finish(ctx, lg, Report{RunID: g.RunID, Layer: LayerDocker, Trigger: trigger, Ring: 0, VMID: vmid,
Mode: "apply", Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
return p
}
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
}
return p
}
// dockerOpts is a signed Docker step (DockerStepExecutor); the zero value is ring 0's unsigned "pending-docker".
type dockerOpts struct {
releaseID string
packages []Package
undo bool
signed map[string]string // blob_b64, sig — the wrapper verifies them ITSELF
}
// EnsureLiveRestore is the ONE-TIME step of `09` decision 87: the wrapper merges `"live-restore": true` into the guest's
// daemon.json and RELOADS docker (never a restart, R-835). A no-op when it is already on.
func (l *Leg) EnsureLiveRestore(ctx context.Context, runID string, vmid int) error {
wr, err := l.call(ctx, runID, map[string]any{"release_id": "live-restore", "layer": LayerGuest, "lane": "fast",
"vmid": vmid, "mode": "live-restore-on", "packages": []Package{}})
if err != nil {
return err
}
if wr.refused() {
return fmt.Errorf("refused: %s", wr.Refused)
}
if wr.failed() {
return fmt.Errorf("failed: %s", wr.Failed)
}
l.log().Info("osupdate: live-restore", "vmid", vmid, "result", string(wr.LiveRestore))
return nil
}
// Facts reads the box's versions through the wrapper's read-only facts mode (R-852): host Debian, kernels, held
// packages, taint, the crash guard; guest Debian, Docker engine, containerd, live-restore. Raw JSON, the wrapper's shape.
func (l *Leg) Facts(ctx context.Context, vmid int) (json.RawMessage, error) {
wr, err := l.call(ctx, "facts"+l.now().UTC().Format("150405"), map[string]any{"release_id": "facts", "layer": LayerHost,
"lane": "fast", "vmid": vmid, "mode": "facts", "packages": []Package{}})
if err != nil {
return nil, err
}
if wr.refused() {
return nil, fmt.Errorf("facts refused: %s", wr.Refused)
}
if len(wr.Facts) == 0 {
return nil, fmt.Errorf("facts: the wrapper returned none (an older wrapper?)")
}
return wr.Facts, nil
}
// runFast is the guest + host fast lane (agent v0.141.x behaviour).
func (l *Leg) runFast(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
runID := l.now().UTC().Format("20060102T150405Z")
lg := l.log().With("run", runID, "vmid", vmid, "trigger", trigger)
if trigger == "night" && l.StatePath != "" {
@@ -338,7 +473,7 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report,
}
}
blk := l.Block()
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk)
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk, dockerOpts{})
if trigger == "night" && l.StatePath != "" {
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
}
@@ -348,20 +483,27 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report,
case !(guest.Outcome == "applied" || guest.Outcome == "nothing" || guest.Outcome == "inventory") || !guest.Healthy:
lg.Warn("osupdate: host step skipped — the guest step did not end healthy", "guest_outcome", guest.Outcome, "reason", guest.HealthReason)
default:
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk)
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk, dockerOpts{})
}
return guest, host
}
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate, do dockerOpts) Report {
rel := hub.WireOSRelease{ID: "ring0-" + runID}
var wire *hub.WireOSRelease
if layer == LayerGuest {
switch layer {
case LayerGuest:
wire = blk.Release
} else {
case LayerHost:
wire = blk.HostRelease
}
if blk.Ring == 1 {
lane := "fast"
if layer == LayerDocker {
lane = "slow"
if do.signed != nil {
rel = hub.WireOSRelease{ID: do.releaseID}
}
} else if blk.Ring == 1 {
rel = hub.WireOSRelease{}
if wire != nil {
rel = *wire
@@ -371,13 +513,23 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
lg := l.log().With("run", runID, "layer", layer, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": "fast", "vmid": vmid, "snapshot": rel.Snapshot,
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": lane, "vmid": vmid, "snapshot": rel.Snapshot,
"packages": []Package{}, "mode": "apply", "select": "listed"}
if rel.ID == "" {
plan["release_id"] = "none"
}
planned := map[string]bool{}
switch {
case layer == LayerDocker && do.signed != nil:
plan["packages"], plan["signed"] = do.packages, do.signed
if do.undo {
plan["undo"] = true
}
for _, p := range do.packages {
planned[p.Name] = true
}
case layer == LayerDocker:
plan["select"] = "pending-docker" // ring 0: the wrapper checks the box's ROOT-OWNED ring-0 mark itself
case !blk.Enabled:
plan["mode"] = "inventory"
lg.Info("osupdate: switched OFF for this box — reporting only")
@@ -406,6 +558,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
rep.Outcome, rep.Refused = "failed", wr.Failed
}
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
if rep.Outcome == "" {
switch {
case rep.Mode == "inventory" && !blk.Enabled:
@@ -423,7 +576,16 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
}
// Health: compare with the start of the pass; give restarted services time (only after an install).
cur := wr.HealthAfter
wantEngine := ""
for _, u := range wr.Upgraded {
if u.Name == "docker-ce" {
wantEngine = EngineOf(u.Version)
}
}
verdict := func(h *Health) (bool, string) {
if layer == LayerDocker {
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
}
if layer == LayerHost {
t := hub.TunnelUnknown
if l.Tunnel != nil {
@@ -449,7 +611,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
break
}
l.sleep(ctx, poll)
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": "fast", "vmid": vmid, "mode": "health", "packages": []Package{}}
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": lane, "vmid": vmid, "mode": "health", "packages": []Package{}}
hr, herr := l.call(ctx, runID, hp)
if herr == nil && hr.Health != nil {
cur = hr.Health
@@ -464,10 +626,36 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
rep.Installed, rep.Pending = wr.Installed, wr.Pending
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
rep.RebootScanned = wr.RebootScanned
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
if layer == LayerDocker {
// the docker report carries the engine set only (the guest report already carries the Debian packages)
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
rep.NotCovered = nil
} else {
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
}
return l.finish(ctx, lg, rep)
}
func onlyDocker(in []Package) []Package {
var out []Package
for _, p := range in {
if DockerNames[p.Name] {
out = append(out, p)
}
}
return out
}
func onlyDockerPending(in []Pending) []Pending {
var out []Pending
for _, p := range in {
if DockerNames[p.Name] {
out = append(out, p)
}
}
return out
}
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in ring 1
// also every fast-lane update the release did not name.
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {