Docker engine slow lane (live-restore once by reload; ring-0 pending-docker under a root-owned ring-0 mark; ring 1 and undo only by a signed os_docker_step the wrapper re-verifies against a root-owned signers file; same-container-id health), the version report (facts mode -> host report system stanza, R-852), guest restart scan every pass (R-849), the crash guard (kernel.panic=10, the 3rd unclean stop in 60 min stays off, 24 h re-arm)
gates / gates (push) Successful in 19s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-04 16:14:36 +02:00
parent 2e2e8f56b8
commit b1746c25af
17 changed files with 1780 additions and 135 deletions
+88
View File
@@ -0,0 +1,88 @@
package osupdate
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)
// OpDockerStep is the signed op class of a Docker engine step (`11` §5.8): a ring-1 box takes an approved engine set,
// and every box takes an UNDO, only through it. CC may sign it until the first paying customer (R-530 ruling).
const OpDockerStep = "os_docker_step"
// DockerStepParams are the signed params. The wrapper compares Packages and Undo with the plan byte-for-byte.
type DockerStepParams struct {
ReleaseID string `json:"release_id"`
Packages []Package `json:"packages"`
Undo bool `json:"undo"`
VMID int `json:"vmid,omitempty"`
}
// DockerStepExecutor runs a verified os_docker_step (signedjobs.Executor). Guest finds the box's customer guest when
// the params name none; Gate (optional) takes the host-wide heavy-op gate so a step never runs beside a backup.
type DockerStepExecutor struct {
Leg *Leg
Guest func(ctx context.Context) (int, error)
Gate func(ctx context.Context) (release func(), err error)
}
// Execute implements signedjobs.Executor.
func (e DockerStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
if op != OpDockerStep {
return signedjobs.ErrNoExecutor
}
so, ok := signedjobs.SignedOpFrom(ctx)
if !ok {
return fmt.Errorf("os_docker_step: no signed envelope in the context — the wrapper could not verify it")
}
var p DockerStepParams
if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 {
return fmt.Errorf("os_docker_step: params must name the engine set: %v", err)
}
vmid := p.VMID
if vmid == 0 {
if e.Guest == nil {
return fmt.Errorf("os_docker_step: no vmid and no guest finder")
}
v, err := e.Guest(ctx)
if err != nil {
return fmt.Errorf("os_docker_step: find the customer guest: %w", err)
}
vmid = v
}
if e.Gate != nil {
release, err := e.Gate(ctx)
if err != nil {
return fmt.Errorf("os_docker_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
}
defer release()
}
rep := e.Leg.RunDockerSigned(ctx, vmid, p, so.Blob, string(so.Sig))
switch rep.Outcome {
case "applied", "nothing":
if rep.Healthy {
return nil
}
}
return fmt.Errorf("os_docker_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
}
// RunDockerSigned is one signed Docker step (ring 1 or an undo): live-restore first (decision 87, a no-op when on),
// then the docker layer with the signed envelope, which the wrapper verifies itself.
func (l *Leg) RunDockerSigned(ctx context.Context, vmid int, p DockerStepParams, blob []byte, sig string) Report {
runID := l.now().UTC().Format("20060102T150405Z")
lg := l.log().With("run", runID, "vmid", vmid, "trigger", "signed", "release", p.ReleaseID, "undo", p.Undo)
if err := l.EnsureLiveRestore(ctx, runID, vmid); err != nil {
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerDocker, Trigger: "signed", Ring: l.Block().Ring, VMID: vmid,
Mode: "apply", ReleaseID: p.ReleaseID, Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
}
rid := p.ReleaseID
if rid == "" {
rid = "signed-" + runID
}
return l.runLayer(ctx, runID, LayerDocker, vmid, "signed", l.Block(), dockerOpts{releaseID: rid, packages: p.Packages,
undo: p.Undo, signed: map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}})
}
+49
View File
@@ -0,0 +1,49 @@
package osupdate
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"testing"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)
// The executor hands the RAW signed bytes to the wrapper (which verifies them itself) and the exact signed package
// list. Red-proof: drop the `signed` field from the docker plan in runLayer and the plan check fails.
func TestDockerStepExecutor_PassesTheSignedEnvelope(t *testing.T) {
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}}, DockerEngine: "29.8.2", Authority: "signed"}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
e := DockerStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
params, _ := json.Marshal(DockerStepParams{ReleaseID: "os-docker-1", Packages: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker CE"}}})
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_docker_step"}`), Sig: []byte("SIG")})
if err := e.Execute(ctx, OpDockerStep, params); err != nil {
t.Fatal(err)
}
dp := w.plans[len(w.plans)-1]
sg, _ := dp["signed"].(map[string]any)
if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["release_id"] != "os-docker-1" || sg == nil ||
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_docker_step"}`)) || sg["sig"] != "SIG" {
t.Fatalf("docker plan = %v", dp)
}
if calls(w) != "guest:live-restore-on,docker:apply" || len(h.reports) != 1 || h.reports[0].Trigger != "signed" {
t.Fatalf("calls=%s reports=%+v", calls(w), h.reports)
}
}
func TestDockerStepExecutor_RefusesWithoutEnvelopeAndPassesOtherOps(t *testing.T) {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
e := DockerStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
if err := e.Execute(context.Background(), "agent_update", nil); !errors.Is(err, signedjobs.ErrNoExecutor) {
t.Fatalf("another op must pass through the chain: %v", err)
}
params, _ := json.Marshal(DockerStepParams{Packages: []Package{{Name: "docker-ce", Version: "1"}}})
if err := e.Execute(context.Background(), OpDockerStep, params); err == nil || len(w.plans) != 0 {
t.Fatalf("no envelope must refuse before any wrapper call: err=%v calls=%s", err, calls(w))
}
}
+204 -16
View File
@@ -1,5 +1,7 @@
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3): the customer GUEST's Debian fast
// lane (agent v0.140.0) and, after it in the same pass, the HOST's (agent v0.141.0).
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3, §5.8): the customer GUEST's Debian
// fast lane (agent v0.140.0), after it in the same pass the HOST's (agent v0.141.0), and then — ring 0 only — the
// guest's DOCKER engine set, the slow lane (agent v0.142.0; a ring-1 box takes a Docker step only inside a signed
// operator job, DockerStepExecutor). It also reads the box's versions for the hub's System page (Facts, R-852).
//
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
@@ -35,10 +37,15 @@ const DefaultPlanDir = "/var/lib/felhom-agent/os"
// Layers.
const (
LayerGuest = "guest"
LayerHost = "host"
LayerGuest = "guest"
LayerHost = "host"
LayerDocker = "docker" // the guest's Docker engine set — slow lane (`11` §5.8)
)
// DockerNames are the six packages of the Docker engine set (the wrapper's DOCKER_NAMES).
var DockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
// Package is one name=version with its origin.
type Package struct {
Name string `json:"name"`
@@ -58,6 +65,7 @@ type Pending struct {
type Container struct {
State string `json:"state"`
Health string `json:"health"` // healthy | unhealthy | starting | none
ID string `json:"id,omitempty"`
}
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
@@ -89,6 +97,11 @@ type WrapperReport struct {
HealthAfter *Health `json:"health_after"`
Health *Health `json:"health"`
PassSeconds float64 `json:"pass_seconds"`
DockerEngine string `json:"docker_engine"`
Authority string `json:"authority"`
Undo bool `json:"undo"`
LiveRestore json.RawMessage `json:"live_restore"`
Facts json.RawMessage `json:"facts"`
}
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
@@ -116,6 +129,9 @@ type Report struct {
RebootScanned bool `json:"reboot_scanned,omitempty"` // the pass looked (host: every pass) — a false RebootNeeded then means "not needed"
Refused json.RawMessage `json:"refused,omitempty"`
PassSeconds float64 `json:"pass_seconds,omitempty"`
DockerEngine string `json:"docker_engine,omitempty"` // docker layer: the engine after the step
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo (downgrade)
}
// Reporter posts a report to the hub (*hub.Client).
@@ -284,6 +300,49 @@ func HostHealthVerdict(before, after *Health, tunnel string) (bool, string) {
return true, ""
}
// EngineOf is the engine version `docker version` prints for a docker-ce package version: "5:29.8.2-1~debian.13~trixie"
// → "29.8.2" (no epoch, no Debian revision).
func EngineOf(pkgVersion string) string {
v := pkgVersion
if i := strings.Index(v, ":"); i >= 0 {
v = v[i+1:]
}
if i := strings.Index(v, "-"); i >= 0 {
v = v[:i]
}
return v
}
// DockerHealthVerdict is THE Docker-step health rule (`11` §5.8; pinned by TestDockerHealthVerdict): the guest rule,
// plus every container running at the start still runs as the SAME container (same id — a changed id means the
// household's apps restarted, which `live-restore` exists to prevent), plus the engine now reports the version the step
// installed (wantEngine "" = no engine change expected).
func DockerHealthVerdict(before, after *Health, wantEngine, gotEngine string) (bool, string) {
if ok, why := HealthVerdict(before, after); !ok {
return false, why
}
if before != nil {
names := make([]string, 0, len(before.Containers))
for n := range before.Containers {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
b := before.Containers[n]
if b.State != "running" || b.ID == "" {
continue
}
if a := after.Containers[n]; a.ID != b.ID {
return false, n + " is a new container (id changed) — the engine step restarted it"
}
}
}
if wantEngine != "" && gotEngine != wantEngine {
return false, "the engine is " + gotEngine + ", not " + wantEngine
}
return true, ""
}
// call writes the plan and runs the wrapper once.
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
dir := l.PlanDir
@@ -320,9 +379,85 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
}
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer. Returns both
// reports (host empty when skipped). trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
// Pass is one leg's reports; an empty Layer means the step did not run.
type Pass struct {
Guest, Host, Docker Report
}
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
// only, after good earlier steps) the Docker engine set. trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
g, h := l.runFast(ctx, vmid, trigger)
p := Pass{Guest: g, Host: h}
if g.Outcome == "skipped" {
return p
}
blk := l.Block()
okStep := func(r Report) bool {
return (r.Outcome == "applied" || r.Outcome == "nothing" || r.Outcome == "inventory") && r.Healthy
}
lg := l.log().With("run", g.RunID, "vmid", vmid, "trigger", trigger)
switch {
case blk.Ring != 0 || !blk.Enabled:
lg.Info("osupdate: docker step skipped — ring 1 takes an engine set only inside a signed operator job (`11` §5.8)", "ring", blk.Ring, "enabled", blk.Enabled)
case !okStep(g) || (h.Layer != "" && !okStep(h)):
lg.Warn("osupdate: docker step skipped — an earlier step did not end healthy")
default:
if err := l.EnsureLiveRestore(ctx, g.RunID, vmid); err != nil {
p.Docker = l.finish(ctx, lg, Report{RunID: g.RunID, Layer: LayerDocker, Trigger: trigger, Ring: 0, VMID: vmid,
Mode: "apply", Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
return p
}
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
}
return p
}
// dockerOpts is a signed Docker step (DockerStepExecutor); the zero value is ring 0's unsigned "pending-docker".
type dockerOpts struct {
releaseID string
packages []Package
undo bool
signed map[string]string // blob_b64, sig — the wrapper verifies them ITSELF
}
// EnsureLiveRestore is the ONE-TIME step of `09` decision 87: the wrapper merges `"live-restore": true` into the guest's
// daemon.json and RELOADS docker (never a restart, R-835). A no-op when it is already on.
func (l *Leg) EnsureLiveRestore(ctx context.Context, runID string, vmid int) error {
wr, err := l.call(ctx, runID, map[string]any{"release_id": "live-restore", "layer": LayerGuest, "lane": "fast",
"vmid": vmid, "mode": "live-restore-on", "packages": []Package{}})
if err != nil {
return err
}
if wr.refused() {
return fmt.Errorf("refused: %s", wr.Refused)
}
if wr.failed() {
return fmt.Errorf("failed: %s", wr.Failed)
}
l.log().Info("osupdate: live-restore", "vmid", vmid, "result", string(wr.LiveRestore))
return nil
}
// Facts reads the box's versions through the wrapper's read-only facts mode (R-852): host Debian, kernels, held
// packages, taint, the crash guard; guest Debian, Docker engine, containerd, live-restore. Raw JSON, the wrapper's shape.
func (l *Leg) Facts(ctx context.Context, vmid int) (json.RawMessage, error) {
wr, err := l.call(ctx, "facts"+l.now().UTC().Format("150405"), map[string]any{"release_id": "facts", "layer": LayerHost,
"lane": "fast", "vmid": vmid, "mode": "facts", "packages": []Package{}})
if err != nil {
return nil, err
}
if wr.refused() {
return nil, fmt.Errorf("facts refused: %s", wr.Refused)
}
if len(wr.Facts) == 0 {
return nil, fmt.Errorf("facts: the wrapper returned none (an older wrapper?)")
}
return wr.Facts, nil
}
// runFast is the guest + host fast lane (agent v0.141.x behaviour).
func (l *Leg) runFast(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
runID := l.now().UTC().Format("20060102T150405Z")
lg := l.log().With("run", runID, "vmid", vmid, "trigger", trigger)
if trigger == "night" && l.StatePath != "" {
@@ -338,7 +473,7 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report,
}
}
blk := l.Block()
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk)
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk, dockerOpts{})
if trigger == "night" && l.StatePath != "" {
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
}
@@ -348,20 +483,27 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report,
case !(guest.Outcome == "applied" || guest.Outcome == "nothing" || guest.Outcome == "inventory") || !guest.Healthy:
lg.Warn("osupdate: host step skipped — the guest step did not end healthy", "guest_outcome", guest.Outcome, "reason", guest.HealthReason)
default:
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk)
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk, dockerOpts{})
}
return guest, host
}
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate, do dockerOpts) Report {
rel := hub.WireOSRelease{ID: "ring0-" + runID}
var wire *hub.WireOSRelease
if layer == LayerGuest {
switch layer {
case LayerGuest:
wire = blk.Release
} else {
case LayerHost:
wire = blk.HostRelease
}
if blk.Ring == 1 {
lane := "fast"
if layer == LayerDocker {
lane = "slow"
if do.signed != nil {
rel = hub.WireOSRelease{ID: do.releaseID}
}
} else if blk.Ring == 1 {
rel = hub.WireOSRelease{}
if wire != nil {
rel = *wire
@@ -371,13 +513,23 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
lg := l.log().With("run", runID, "layer", layer, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": "fast", "vmid": vmid, "snapshot": rel.Snapshot,
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": lane, "vmid": vmid, "snapshot": rel.Snapshot,
"packages": []Package{}, "mode": "apply", "select": "listed"}
if rel.ID == "" {
plan["release_id"] = "none"
}
planned := map[string]bool{}
switch {
case layer == LayerDocker && do.signed != nil:
plan["packages"], plan["signed"] = do.packages, do.signed
if do.undo {
plan["undo"] = true
}
for _, p := range do.packages {
planned[p.Name] = true
}
case layer == LayerDocker:
plan["select"] = "pending-docker" // ring 0: the wrapper checks the box's ROOT-OWNED ring-0 mark itself
case !blk.Enabled:
plan["mode"] = "inventory"
lg.Info("osupdate: switched OFF for this box — reporting only")
@@ -406,6 +558,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
rep.Outcome, rep.Refused = "failed", wr.Failed
}
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
if rep.Outcome == "" {
switch {
case rep.Mode == "inventory" && !blk.Enabled:
@@ -423,7 +576,16 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
}
// Health: compare with the start of the pass; give restarted services time (only after an install).
cur := wr.HealthAfter
wantEngine := ""
for _, u := range wr.Upgraded {
if u.Name == "docker-ce" {
wantEngine = EngineOf(u.Version)
}
}
verdict := func(h *Health) (bool, string) {
if layer == LayerDocker {
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
}
if layer == LayerHost {
t := hub.TunnelUnknown
if l.Tunnel != nil {
@@ -449,7 +611,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
break
}
l.sleep(ctx, poll)
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": "fast", "vmid": vmid, "mode": "health", "packages": []Package{}}
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": lane, "vmid": vmid, "mode": "health", "packages": []Package{}}
hr, herr := l.call(ctx, runID, hp)
if herr == nil && hr.Health != nil {
cur = hr.Health
@@ -464,10 +626,36 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
rep.Installed, rep.Pending = wr.Installed, wr.Pending
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
rep.RebootScanned = wr.RebootScanned
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
if layer == LayerDocker {
// the docker report carries the engine set only (the guest report already carries the Debian packages)
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
rep.NotCovered = nil
} else {
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
}
return l.finish(ctx, lg, rep)
}
func onlyDocker(in []Package) []Package {
var out []Package
for _, p := range in {
if DockerNames[p.Name] {
out = append(out, p)
}
}
return out
}
func onlyDockerPending(in []Pending) []Pending {
var out []Pending
for _, p := range in {
if DockerNames[p.Name] {
out = append(out, p)
}
}
return out
}
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in ring 1
// also every fast-lane update the release did not name.
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
+125 -27
View File
@@ -134,14 +134,14 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "u3"}}},
}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
g, ho := l.Run(context.Background(), 9201, "night")
g, ho := run2(l, "night")
if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
t.Fatalf("guest %+v\nhost %+v", g, ho)
}
if calls(w) != "guest:apply,host:apply" {
t.Fatalf("calls = %s, want one apply per layer, guest first", calls(w))
if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply" {
t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore and the ring-0 docker step", calls(w))
}
for _, p := range w.plans {
for _, p := range w.plans[:2] {
if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
t.Fatalf("ring-0 plan = %v", p)
}
@@ -149,7 +149,7 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
t.Fatalf("not covered = %v", g.NotCovered)
}
if len(h.reports) != 2 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost {
if len(h.reports) != 3 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker {
t.Fatalf("hub got %+v", h.reports)
}
}
@@ -163,7 +163,7 @@ func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}}
hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr})
g, ho := l.Run(context.Background(), 9201, "night")
g, ho := run2(l, "night")
if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" {
t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID)
}
@@ -182,7 +182,7 @@ func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
func TestRing1_NoReleaseIsInventory(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
g, ho := l.Run(context.Background(), 9201, "night")
g, ho := run2(l, "night")
if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" {
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
}
@@ -192,7 +192,7 @@ func TestRing1_NoReleaseIsInventory(t *testing.T) {
func TestNoBlock_IsRing1Nothing(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, _ := newLeg(t, w, nil)
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "nothing" || g.Ring != 1 {
if g, _ := run2(l, "night"); g.Outcome != "nothing" || g.Ring != 1 {
t.Fatalf("g=%+v calls=%s", g, calls(w))
}
}
@@ -201,7 +201,7 @@ func TestNoBlock_IsRing1Nothing(t *testing.T) {
func TestSwitchOff_ReportsOnly(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
g, ho := l.Run(context.Background(), 9201, "night")
g, ho := run2(l, "night")
if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 {
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
}
@@ -212,8 +212,9 @@ func TestBYO_NoHostPlan(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Appliance = false
_, ho := l.Run(context.Background(), 9201, "night")
if ho.Outcome != "" || calls(w) != "guest:apply" || len(h.reports) != 1 {
_, ho := run2(l, "night")
// the guest (and so its Docker engine) is ours on a BYO box too: only the HOST is the owner's
if ho.Outcome != "" || calls(w) != "guest:apply,guest:live-restore-on,docker:apply" || len(h.reports) != 2 {
t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w))
}
}
@@ -223,7 +224,7 @@ func TestGuestFailure_SkipsTheHost(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
g, ho := l.Run(context.Background(), 9201, "night")
g, ho := run2(l, "night")
if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" {
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
}
@@ -236,7 +237,7 @@ func TestHealth_FailsAfterTheWait(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}},
healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
g, ho := l.Run(context.Background(), 9201, "night")
g, ho := run2(l, "night")
if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" {
t.Fatalf("g=%+v h=%+v", g, ho)
}
@@ -251,7 +252,7 @@ func TestHealth_RecoversInsideTheWait(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}},
healthSeq: map[string][]*Health{LayerGuest: {starting}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "applied" || !g.Healthy {
if g, _ := run2(l, "night"); g.Outcome != "applied" || !g.Healthy {
t.Fatalf("g = %+v", g)
}
}
@@ -261,7 +262,7 @@ func TestHost_TunnelDownFailsTheHostStep(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Tunnel = fakeTunnel{hub.TunnelNotRunning}
_, ho := l.Run(context.Background(), 9201, "night")
_, ho := run2(l, "night")
if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") {
t.Fatalf("host = %+v", ho)
}
@@ -328,12 +329,12 @@ func TestHostHealthVerdict(t *testing.T) {
func TestOncePerNight(t *testing.T) {
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "night")
run2(l, "night")
n := len(w.plans)
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "skipped" || len(w.plans) != n {
if g, _ := run2(l, "night"); g.Outcome != "skipped" || len(w.plans) != n {
t.Fatalf("a second night run in the same night ran: %+v", g)
}
if g, _ := l.Run(context.Background(), 9201, "debug"); g.Outcome == "skipped" {
if g, _ := run2(l, "debug"); g.Outcome == "skipped" {
t.Fatal("the debug action must not be throttled")
}
}
@@ -344,13 +345,16 @@ func TestWrapperSuite(t *testing.T) {
if err != nil {
t.Skip("python3 not available")
}
cmd := exec.Command(py, "-B", "../../configs/test_felhom_os_apply.py")
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("wrapper suite failed: %v\n%s", err, out)
}
if !strings.Contains(string(out), "OK") {
t.Fatalf("wrapper suite did not report OK:\n%s", out)
// the OS wrapper and (agent v0.142.0) the crash guard — both root programs in configs/ with their own suites
for _, suite := range []string{"../../configs/test_felhom_os_apply.py", "../../configs/test_felhom_crash_guard.py"} {
cmd := exec.Command(py, "-B", suite)
out, err := cmd.CombinedOutput()
if err != nil {
t.Fatalf("%s failed: %v\n%s", suite, err, out)
}
if !strings.Contains(string(out), "OK") {
t.Fatalf("%s did not report OK:\n%s", suite, out)
}
}
}
@@ -362,8 +366,102 @@ func TestHostReport_CarriesRebootScanned(t *testing.T) {
LayerHost: {RebootScanned: true, RebootNeeded: true, RestartNeeded: []string{"lxc-start"}},
}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "night")
if len(h.reports) != 2 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
run2(l, "night")
if len(h.reports) != 3 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
t.Fatalf("hub got %+v", h.reports)
}
}
// run2 is the guest + host reports of one pass (the tests written before the docker step).
func run2(l *Leg, trigger string) (Report, Report) {
p := l.Run(context.Background(), 9201, trigger)
return p.Guest, p.Host
}
// ---- the Docker step (`11` §5.8, agent v0.142.0) ----
// Ring 1 never takes an engine step in the night leg — only inside a signed operator job. Red-proof: drop the
// `blk.Ring != 0` case in Run and the ring-1 pass makes a docker call.
func TestDocker_Ring1NightLegNeverSteps(t *testing.T) {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") || strings.Contains(calls(w), "live-restore") {
t.Fatalf("ring 1 took a docker step: %s", calls(w))
}
}
// An unhealthy earlier step skips the docker step.
func TestDocker_SkippedAfterAnUnhealthyStep(t *testing.T) {
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}},
healthSeq: map[string][]*Health{}}
bad := guestOK()
bad.Controller = "unhealthy"
w.applyRep[LayerGuest] = WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}
w.healthSeq[LayerGuest] = []*Health{bad, bad, bad, bad, bad, bad, bad, bad}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") {
t.Fatalf("docker step ran after an unhealthy guest step: %s", calls(w))
}
}
// The docker plan is the slow lane, pending-docker for ring 0; the report carries only the engine set.
func TestDocker_Ring0PlanAndReport(t *testing.T) {
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}},
Installed: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"}, {Name: "libc6", Version: "u4", Origin: "Debian"}},
DockerEngine: "29.8.2", Authority: "ring0"}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
dp := w.plans[len(w.plans)-1]
if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["select"] != "pending-docker" {
t.Fatalf("docker plan = %v", dp)
}
d := p.Docker
if d.Outcome != "applied" || !d.Healthy || d.DockerEngine != "29.8.2" || len(d.Installed) != 1 || d.Installed[0].Name != "docker-ce" {
t.Fatalf("docker report = %+v", d)
}
}
// THE docker health rule. Red-proof: drop the id comparison (or the engine check) in DockerHealthVerdict and a case fails.
func TestDockerHealthVerdict(t *testing.T) {
before := guestOK()
before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
"app": {State: "running", Health: "healthy", ID: "b"}}
same := guestOK()
same.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
"app": {State: "running", Health: "healthy", ID: "b"}}
moved := guestOK()
moved.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
"app": {State: "running", Health: "healthy", ID: "c"}}
if ok, why := DockerHealthVerdict(before, same, "29.8.2", "29.8.2"); !ok {
t.Fatalf("same ids, right engine: %s", why)
}
if ok, _ := DockerHealthVerdict(before, moved, "29.8.2", "29.8.2"); ok {
t.Fatal("a changed container id passed — live-restore failed and the apps restarted")
}
if ok, _ := DockerHealthVerdict(before, same, "29.8.2", "29.7.2"); ok {
t.Fatal("the engine did not move and the step passed")
}
if EngineOf("5:29.8.2-1~debian.13~trixie") != "29.8.2" {
t.Fatalf("EngineOf = %q", EngineOf("5:29.8.2-1~debian.13~trixie"))
}
}
// A changed id after the step → health_failed (the consequence, not only the verdict).
func TestDocker_ChangedIDIsHealthFailed(t *testing.T) {
before := guestOK()
before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}}
after := guestOK()
after.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "z"}}
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1"}}, DockerEngine: "29.8.2",
HealthBefore: before, HealthAfter: after}}, healthSeq: map[string][]*Health{}}
w.healthSeq[LayerDocker] = []*Health{after, after, after, after, after, after, after, after}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
if p.Docker.Outcome != "health_failed" || !strings.Contains(p.Docker.HealthReason, "id changed") {
t.Fatalf("docker = %+v", p.Docker)
}
}