Docker engine slow lane (live-restore once by reload; ring-0 pending-docker under a root-owned ring-0 mark; ring 1 and undo only by a signed os_docker_step the wrapper re-verifies against a root-owned signers file; same-container-id health), the version report (facts mode -> host report system stanza, R-852), guest restart scan every pass (R-849), the crash guard (kernel.panic=10, the 3rd unclean stop in 60 min stays off, 24 h re-arm)
gates / gates (push) Successful in 19s
gates / gates (push) Successful in 19s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
@@ -0,0 +1,88 @@
|
||||
package osupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||
)
|
||||
|
||||
// OpDockerStep is the signed op class of a Docker engine step (`11` §5.8): a ring-1 box takes an approved engine set,
|
||||
// and every box takes an UNDO, only through it. CC may sign it until the first paying customer (R-530 ruling).
|
||||
const OpDockerStep = "os_docker_step"
|
||||
|
||||
// DockerStepParams are the signed params. The wrapper compares Packages and Undo with the plan byte-for-byte.
|
||||
type DockerStepParams struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Packages []Package `json:"packages"`
|
||||
Undo bool `json:"undo"`
|
||||
VMID int `json:"vmid,omitempty"`
|
||||
}
|
||||
|
||||
// DockerStepExecutor runs a verified os_docker_step (signedjobs.Executor). Guest finds the box's customer guest when
|
||||
// the params name none; Gate (optional) takes the host-wide heavy-op gate so a step never runs beside a backup.
|
||||
type DockerStepExecutor struct {
|
||||
Leg *Leg
|
||||
Guest func(ctx context.Context) (int, error)
|
||||
Gate func(ctx context.Context) (release func(), err error)
|
||||
}
|
||||
|
||||
// Execute implements signedjobs.Executor.
|
||||
func (e DockerStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
|
||||
if op != OpDockerStep {
|
||||
return signedjobs.ErrNoExecutor
|
||||
}
|
||||
so, ok := signedjobs.SignedOpFrom(ctx)
|
||||
if !ok {
|
||||
return fmt.Errorf("os_docker_step: no signed envelope in the context — the wrapper could not verify it")
|
||||
}
|
||||
var p DockerStepParams
|
||||
if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 {
|
||||
return fmt.Errorf("os_docker_step: params must name the engine set: %v", err)
|
||||
}
|
||||
vmid := p.VMID
|
||||
if vmid == 0 {
|
||||
if e.Guest == nil {
|
||||
return fmt.Errorf("os_docker_step: no vmid and no guest finder")
|
||||
}
|
||||
v, err := e.Guest(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("os_docker_step: find the customer guest: %w", err)
|
||||
}
|
||||
vmid = v
|
||||
}
|
||||
if e.Gate != nil {
|
||||
release, err := e.Gate(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("os_docker_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
|
||||
}
|
||||
defer release()
|
||||
}
|
||||
rep := e.Leg.RunDockerSigned(ctx, vmid, p, so.Blob, string(so.Sig))
|
||||
switch rep.Outcome {
|
||||
case "applied", "nothing":
|
||||
if rep.Healthy {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("os_docker_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
|
||||
}
|
||||
|
||||
// RunDockerSigned is one signed Docker step (ring 1 or an undo): live-restore first (decision 87, a no-op when on),
|
||||
// then the docker layer with the signed envelope, which the wrapper verifies itself.
|
||||
func (l *Leg) RunDockerSigned(ctx context.Context, vmid int, p DockerStepParams, blob []byte, sig string) Report {
|
||||
runID := l.now().UTC().Format("20060102T150405Z")
|
||||
lg := l.log().With("run", runID, "vmid", vmid, "trigger", "signed", "release", p.ReleaseID, "undo", p.Undo)
|
||||
if err := l.EnsureLiveRestore(ctx, runID, vmid); err != nil {
|
||||
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerDocker, Trigger: "signed", Ring: l.Block().Ring, VMID: vmid,
|
||||
Mode: "apply", ReleaseID: p.ReleaseID, Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
|
||||
}
|
||||
rid := p.ReleaseID
|
||||
if rid == "" {
|
||||
rid = "signed-" + runID
|
||||
}
|
||||
return l.runLayer(ctx, runID, LayerDocker, vmid, "signed", l.Block(), dockerOpts{releaseID: rid, packages: p.Packages,
|
||||
undo: p.Undo, signed: map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}})
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package osupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||
)
|
||||
|
||||
// The executor hands the RAW signed bytes to the wrapper (which verifies them itself) and the exact signed package
|
||||
// list. Red-proof: drop the `signed` field from the docker plan in runLayer and the plan check fails.
|
||||
func TestDockerStepExecutor_PassesTheSignedEnvelope(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
|
||||
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}}, DockerEngine: "29.8.2", Authority: "signed"}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
e := DockerStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
|
||||
params, _ := json.Marshal(DockerStepParams{ReleaseID: "os-docker-1", Packages: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker CE"}}})
|
||||
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_docker_step"}`), Sig: []byte("SIG")})
|
||||
if err := e.Execute(ctx, OpDockerStep, params); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dp := w.plans[len(w.plans)-1]
|
||||
sg, _ := dp["signed"].(map[string]any)
|
||||
if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["release_id"] != "os-docker-1" || sg == nil ||
|
||||
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_docker_step"}`)) || sg["sig"] != "SIG" {
|
||||
t.Fatalf("docker plan = %v", dp)
|
||||
}
|
||||
if calls(w) != "guest:live-restore-on,docker:apply" || len(h.reports) != 1 || h.reports[0].Trigger != "signed" {
|
||||
t.Fatalf("calls=%s reports=%+v", calls(w), h.reports)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerStepExecutor_RefusesWithoutEnvelopeAndPassesOtherOps(t *testing.T) {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
e := DockerStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
|
||||
if err := e.Execute(context.Background(), "agent_update", nil); !errors.Is(err, signedjobs.ErrNoExecutor) {
|
||||
t.Fatalf("another op must pass through the chain: %v", err)
|
||||
}
|
||||
params, _ := json.Marshal(DockerStepParams{Packages: []Package{{Name: "docker-ce", Version: "1"}}})
|
||||
if err := e.Execute(context.Background(), OpDockerStep, params); err == nil || len(w.plans) != 0 {
|
||||
t.Fatalf("no envelope must refuse before any wrapper call: err=%v calls=%s", err, calls(w))
|
||||
}
|
||||
}
|
||||
+204
-16
@@ -1,5 +1,7 @@
|
||||
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3): the customer GUEST's Debian fast
|
||||
// lane (agent v0.140.0) and, after it in the same pass, the HOST's (agent v0.141.0).
|
||||
// Package osupdate is the agent's OS-update leg (`11-os-updates.md` §8 steps 2–3, §5.8): the customer GUEST's Debian
|
||||
// fast lane (agent v0.140.0), after it in the same pass the HOST's (agent v0.141.0), and then — ring 0 only — the
|
||||
// guest's DOCKER engine set, the slow lane (agent v0.142.0; a ring-1 box takes a Docker step only inside a signed
|
||||
// operator job, DockerStepExecutor). It also reads the box's versions for the hub's System page (Facts, R-852).
|
||||
//
|
||||
// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
|
||||
// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
|
||||
@@ -35,10 +37,15 @@ const DefaultPlanDir = "/var/lib/felhom-agent/os"
|
||||
|
||||
// Layers.
|
||||
const (
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
LayerDocker = "docker" // the guest's Docker engine set — slow lane (`11` §5.8)
|
||||
)
|
||||
|
||||
// DockerNames are the six packages of the Docker engine set (the wrapper's DOCKER_NAMES).
|
||||
var DockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
|
||||
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
|
||||
|
||||
// Package is one name=version with its origin.
|
||||
type Package struct {
|
||||
Name string `json:"name"`
|
||||
@@ -58,6 +65,7 @@ type Pending struct {
|
||||
type Container struct {
|
||||
State string `json:"state"`
|
||||
Health string `json:"health"` // healthy | unhealthy | starting | none
|
||||
ID string `json:"id,omitempty"`
|
||||
}
|
||||
|
||||
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
|
||||
@@ -89,6 +97,11 @@ type WrapperReport struct {
|
||||
HealthAfter *Health `json:"health_after"`
|
||||
Health *Health `json:"health"`
|
||||
PassSeconds float64 `json:"pass_seconds"`
|
||||
DockerEngine string `json:"docker_engine"`
|
||||
Authority string `json:"authority"`
|
||||
Undo bool `json:"undo"`
|
||||
LiveRestore json.RawMessage `json:"live_restore"`
|
||||
Facts json.RawMessage `json:"facts"`
|
||||
}
|
||||
|
||||
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
|
||||
@@ -116,6 +129,9 @@ type Report struct {
|
||||
RebootScanned bool `json:"reboot_scanned,omitempty"` // the pass looked (host: every pass) — a false RebootNeeded then means "not needed"
|
||||
Refused json.RawMessage `json:"refused,omitempty"`
|
||||
PassSeconds float64 `json:"pass_seconds,omitempty"`
|
||||
DockerEngine string `json:"docker_engine,omitempty"` // docker layer: the engine after the step
|
||||
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
|
||||
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo (downgrade)
|
||||
}
|
||||
|
||||
// Reporter posts a report to the hub (*hub.Client).
|
||||
@@ -284,6 +300,49 @@ func HostHealthVerdict(before, after *Health, tunnel string) (bool, string) {
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// EngineOf is the engine version `docker version` prints for a docker-ce package version: "5:29.8.2-1~debian.13~trixie"
|
||||
// → "29.8.2" (no epoch, no Debian revision).
|
||||
func EngineOf(pkgVersion string) string {
|
||||
v := pkgVersion
|
||||
if i := strings.Index(v, ":"); i >= 0 {
|
||||
v = v[i+1:]
|
||||
}
|
||||
if i := strings.Index(v, "-"); i >= 0 {
|
||||
v = v[:i]
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// DockerHealthVerdict is THE Docker-step health rule (`11` §5.8; pinned by TestDockerHealthVerdict): the guest rule,
|
||||
// plus every container running at the start still runs as the SAME container (same id — a changed id means the
|
||||
// household's apps restarted, which `live-restore` exists to prevent), plus the engine now reports the version the step
|
||||
// installed (wantEngine "" = no engine change expected).
|
||||
func DockerHealthVerdict(before, after *Health, wantEngine, gotEngine string) (bool, string) {
|
||||
if ok, why := HealthVerdict(before, after); !ok {
|
||||
return false, why
|
||||
}
|
||||
if before != nil {
|
||||
names := make([]string, 0, len(before.Containers))
|
||||
for n := range before.Containers {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
b := before.Containers[n]
|
||||
if b.State != "running" || b.ID == "" {
|
||||
continue
|
||||
}
|
||||
if a := after.Containers[n]; a.ID != b.ID {
|
||||
return false, n + " is a new container (id changed) — the engine step restarted it"
|
||||
}
|
||||
}
|
||||
}
|
||||
if wantEngine != "" && gotEngine != wantEngine {
|
||||
return false, "the engine is " + gotEngine + ", not " + wantEngine
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// call writes the plan and runs the wrapper once.
|
||||
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
|
||||
dir := l.PlanDir
|
||||
@@ -320,9 +379,85 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
|
||||
return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
|
||||
}
|
||||
|
||||
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer. Returns both
|
||||
// reports (host empty when skipped). trigger is "night" or "debug".
|
||||
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
|
||||
// Pass is one leg's reports; an empty Layer means the step did not run.
|
||||
type Pass struct {
|
||||
Guest, Host, Docker Report
|
||||
}
|
||||
|
||||
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
|
||||
// only, after good earlier steps) the Docker engine set. trigger is "night" or "debug".
|
||||
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
|
||||
g, h := l.runFast(ctx, vmid, trigger)
|
||||
p := Pass{Guest: g, Host: h}
|
||||
if g.Outcome == "skipped" {
|
||||
return p
|
||||
}
|
||||
blk := l.Block()
|
||||
okStep := func(r Report) bool {
|
||||
return (r.Outcome == "applied" || r.Outcome == "nothing" || r.Outcome == "inventory") && r.Healthy
|
||||
}
|
||||
lg := l.log().With("run", g.RunID, "vmid", vmid, "trigger", trigger)
|
||||
switch {
|
||||
case blk.Ring != 0 || !blk.Enabled:
|
||||
lg.Info("osupdate: docker step skipped — ring 1 takes an engine set only inside a signed operator job (`11` §5.8)", "ring", blk.Ring, "enabled", blk.Enabled)
|
||||
case !okStep(g) || (h.Layer != "" && !okStep(h)):
|
||||
lg.Warn("osupdate: docker step skipped — an earlier step did not end healthy")
|
||||
default:
|
||||
if err := l.EnsureLiveRestore(ctx, g.RunID, vmid); err != nil {
|
||||
p.Docker = l.finish(ctx, lg, Report{RunID: g.RunID, Layer: LayerDocker, Trigger: trigger, Ring: 0, VMID: vmid,
|
||||
Mode: "apply", Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
|
||||
return p
|
||||
}
|
||||
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// dockerOpts is a signed Docker step (DockerStepExecutor); the zero value is ring 0's unsigned "pending-docker".
|
||||
type dockerOpts struct {
|
||||
releaseID string
|
||||
packages []Package
|
||||
undo bool
|
||||
signed map[string]string // blob_b64, sig — the wrapper verifies them ITSELF
|
||||
}
|
||||
|
||||
// EnsureLiveRestore is the ONE-TIME step of `09` decision 87: the wrapper merges `"live-restore": true` into the guest's
|
||||
// daemon.json and RELOADS docker (never a restart, R-835). A no-op when it is already on.
|
||||
func (l *Leg) EnsureLiveRestore(ctx context.Context, runID string, vmid int) error {
|
||||
wr, err := l.call(ctx, runID, map[string]any{"release_id": "live-restore", "layer": LayerGuest, "lane": "fast",
|
||||
"vmid": vmid, "mode": "live-restore-on", "packages": []Package{}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if wr.refused() {
|
||||
return fmt.Errorf("refused: %s", wr.Refused)
|
||||
}
|
||||
if wr.failed() {
|
||||
return fmt.Errorf("failed: %s", wr.Failed)
|
||||
}
|
||||
l.log().Info("osupdate: live-restore", "vmid", vmid, "result", string(wr.LiveRestore))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Facts reads the box's versions through the wrapper's read-only facts mode (R-852): host Debian, kernels, held
|
||||
// packages, taint, the crash guard; guest Debian, Docker engine, containerd, live-restore. Raw JSON, the wrapper's shape.
|
||||
func (l *Leg) Facts(ctx context.Context, vmid int) (json.RawMessage, error) {
|
||||
wr, err := l.call(ctx, "facts"+l.now().UTC().Format("150405"), map[string]any{"release_id": "facts", "layer": LayerHost,
|
||||
"lane": "fast", "vmid": vmid, "mode": "facts", "packages": []Package{}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if wr.refused() {
|
||||
return nil, fmt.Errorf("facts refused: %s", wr.Refused)
|
||||
}
|
||||
if len(wr.Facts) == 0 {
|
||||
return nil, fmt.Errorf("facts: the wrapper returned none (an older wrapper?)")
|
||||
}
|
||||
return wr.Facts, nil
|
||||
}
|
||||
|
||||
// runFast is the guest + host fast lane (agent v0.141.x behaviour).
|
||||
func (l *Leg) runFast(ctx context.Context, vmid int, trigger string) (guest Report, host Report) {
|
||||
runID := l.now().UTC().Format("20060102T150405Z")
|
||||
lg := l.log().With("run", runID, "vmid", vmid, "trigger", trigger)
|
||||
if trigger == "night" && l.StatePath != "" {
|
||||
@@ -338,7 +473,7 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report,
|
||||
}
|
||||
}
|
||||
blk := l.Block()
|
||||
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk)
|
||||
guest = l.runLayer(ctx, runID, LayerGuest, vmid, trigger, blk, dockerOpts{})
|
||||
if trigger == "night" && l.StatePath != "" {
|
||||
_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
|
||||
}
|
||||
@@ -348,20 +483,27 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) (guest Report,
|
||||
case !(guest.Outcome == "applied" || guest.Outcome == "nothing" || guest.Outcome == "inventory") || !guest.Healthy:
|
||||
lg.Warn("osupdate: host step skipped — the guest step did not end healthy", "guest_outcome", guest.Outcome, "reason", guest.HealthReason)
|
||||
default:
|
||||
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk)
|
||||
host = l.runLayer(ctx, runID, LayerHost, vmid, trigger, blk, dockerOpts{})
|
||||
}
|
||||
return guest, host
|
||||
}
|
||||
|
||||
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate) Report {
|
||||
func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigger string, blk hub.WireOSUpdate, do dockerOpts) Report {
|
||||
rel := hub.WireOSRelease{ID: "ring0-" + runID}
|
||||
var wire *hub.WireOSRelease
|
||||
if layer == LayerGuest {
|
||||
switch layer {
|
||||
case LayerGuest:
|
||||
wire = blk.Release
|
||||
} else {
|
||||
case LayerHost:
|
||||
wire = blk.HostRelease
|
||||
}
|
||||
if blk.Ring == 1 {
|
||||
lane := "fast"
|
||||
if layer == LayerDocker {
|
||||
lane = "slow"
|
||||
if do.signed != nil {
|
||||
rel = hub.WireOSRelease{ID: do.releaseID}
|
||||
}
|
||||
} else if blk.Ring == 1 {
|
||||
rel = hub.WireOSRelease{}
|
||||
if wire != nil {
|
||||
rel = *wire
|
||||
@@ -371,13 +513,23 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
lg := l.log().With("run", runID, "layer", layer, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
|
||||
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
|
||||
|
||||
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": "fast", "vmid": vmid, "snapshot": rel.Snapshot,
|
||||
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": lane, "vmid": vmid, "snapshot": rel.Snapshot,
|
||||
"packages": []Package{}, "mode": "apply", "select": "listed"}
|
||||
if rel.ID == "" {
|
||||
plan["release_id"] = "none"
|
||||
}
|
||||
planned := map[string]bool{}
|
||||
switch {
|
||||
case layer == LayerDocker && do.signed != nil:
|
||||
plan["packages"], plan["signed"] = do.packages, do.signed
|
||||
if do.undo {
|
||||
plan["undo"] = true
|
||||
}
|
||||
for _, p := range do.packages {
|
||||
planned[p.Name] = true
|
||||
}
|
||||
case layer == LayerDocker:
|
||||
plan["select"] = "pending-docker" // ring 0: the wrapper checks the box's ROOT-OWNED ring-0 mark itself
|
||||
case !blk.Enabled:
|
||||
plan["mode"] = "inventory"
|
||||
lg.Info("osupdate: switched OFF for this box — reporting only")
|
||||
@@ -406,6 +558,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
rep.Outcome, rep.Refused = "failed", wr.Failed
|
||||
}
|
||||
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
|
||||
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
|
||||
if rep.Outcome == "" {
|
||||
switch {
|
||||
case rep.Mode == "inventory" && !blk.Enabled:
|
||||
@@ -423,7 +576,16 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
}
|
||||
// Health: compare with the start of the pass; give restarted services time (only after an install).
|
||||
cur := wr.HealthAfter
|
||||
wantEngine := ""
|
||||
for _, u := range wr.Upgraded {
|
||||
if u.Name == "docker-ce" {
|
||||
wantEngine = EngineOf(u.Version)
|
||||
}
|
||||
}
|
||||
verdict := func(h *Health) (bool, string) {
|
||||
if layer == LayerDocker {
|
||||
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
|
||||
}
|
||||
if layer == LayerHost {
|
||||
t := hub.TunnelUnknown
|
||||
if l.Tunnel != nil {
|
||||
@@ -449,7 +611,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
break
|
||||
}
|
||||
l.sleep(ctx, poll)
|
||||
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": "fast", "vmid": vmid, "mode": "health", "packages": []Package{}}
|
||||
hp := map[string]any{"release_id": plan["release_id"], "layer": layer, "lane": lane, "vmid": vmid, "mode": "health", "packages": []Package{}}
|
||||
hr, herr := l.call(ctx, runID, hp)
|
||||
if herr == nil && hr.Health != nil {
|
||||
cur = hr.Health
|
||||
@@ -464,10 +626,36 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
rep.Installed, rep.Pending = wr.Installed, wr.Pending
|
||||
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
|
||||
rep.RebootScanned = wr.RebootScanned
|
||||
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
|
||||
if layer == LayerDocker {
|
||||
// the docker report carries the engine set only (the guest report already carries the Debian packages)
|
||||
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
|
||||
rep.NotCovered = nil
|
||||
} else {
|
||||
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
|
||||
}
|
||||
return l.finish(ctx, lg, rep)
|
||||
}
|
||||
|
||||
func onlyDocker(in []Package) []Package {
|
||||
var out []Package
|
||||
for _, p := range in {
|
||||
if DockerNames[p.Name] {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func onlyDockerPending(in []Pending) []Pending {
|
||||
var out []Pending
|
||||
for _, p := range in {
|
||||
if DockerNames[p.Name] {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in ring 1
|
||||
// also every fast-lane update the release did not name.
|
||||
func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
|
||||
|
||||
+125
-27
@@ -134,14 +134,14 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
|
||||
LayerHost: {Upgraded: []Package{{Name: "openssl", Version: "u3"}}},
|
||||
}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
g, ho := run2(l, "night")
|
||||
if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
|
||||
t.Fatalf("guest %+v\nhost %+v", g, ho)
|
||||
}
|
||||
if calls(w) != "guest:apply,host:apply" {
|
||||
t.Fatalf("calls = %s, want one apply per layer, guest first", calls(w))
|
||||
if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply" {
|
||||
t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore and the ring-0 docker step", calls(w))
|
||||
}
|
||||
for _, p := range w.plans {
|
||||
for _, p := range w.plans[:2] {
|
||||
if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
|
||||
t.Fatalf("ring-0 plan = %v", p)
|
||||
}
|
||||
@@ -149,7 +149,7 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
|
||||
if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
|
||||
t.Fatalf("not covered = %v", g.NotCovered)
|
||||
}
|
||||
if len(h.reports) != 2 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost {
|
||||
if len(h.reports) != 3 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker {
|
||||
t.Fatalf("hub got %+v", h.reports)
|
||||
}
|
||||
}
|
||||
@@ -163,7 +163,7 @@ func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
|
||||
gr := &hub.WireOSRelease{ID: "os-g", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "g-u4", Origin: "Debian"}}}
|
||||
hr := &hub.WireOSRelease{ID: "os-h", Snapshot: "20261004T090000Z", Packages: []hub.WireOSPackage{{Name: "openssl", Version: "h-u3", Origin: "Debian-Security"}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: gr, HostRelease: hr})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
g, ho := run2(l, "night")
|
||||
if g.ReleaseID != "os-g" || ho.ReleaseID != "os-h" {
|
||||
t.Fatalf("release ids %q %q", g.ReleaseID, ho.ReleaseID)
|
||||
}
|
||||
@@ -182,7 +182,7 @@ func TestRing1_EachLayerItsOwnRelease(t *testing.T) {
|
||||
func TestRing1_NoReleaseIsInventory(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
g, ho := run2(l, "night")
|
||||
if g.Outcome != "nothing" || ho.Outcome != "nothing" || calls(w) != "guest:inventory,host:inventory" {
|
||||
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
|
||||
}
|
||||
@@ -192,7 +192,7 @@ func TestRing1_NoReleaseIsInventory(t *testing.T) {
|
||||
func TestNoBlock_IsRing1Nothing(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend}
|
||||
l, _ := newLeg(t, w, nil)
|
||||
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "nothing" || g.Ring != 1 {
|
||||
if g, _ := run2(l, "night"); g.Outcome != "nothing" || g.Ring != 1 {
|
||||
t.Fatalf("g=%+v calls=%s", g, calls(w))
|
||||
}
|
||||
}
|
||||
@@ -201,7 +201,7 @@ func TestNoBlock_IsRing1Nothing(t *testing.T) {
|
||||
func TestSwitchOff_ReportsOnly(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
g, ho := run2(l, "night")
|
||||
if g.Outcome != "inventory" || ho.Outcome != "inventory" || calls(w) != "guest:inventory,host:inventory" || len(h.reports) != 2 {
|
||||
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
|
||||
}
|
||||
@@ -212,8 +212,9 @@ func TestBYO_NoHostPlan(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Appliance = false
|
||||
_, ho := l.Run(context.Background(), 9201, "night")
|
||||
if ho.Outcome != "" || calls(w) != "guest:apply" || len(h.reports) != 1 {
|
||||
_, ho := run2(l, "night")
|
||||
// the guest (and so its Docker engine) is ours on a BYO box too: only the HOST is the owner's
|
||||
if ho.Outcome != "" || calls(w) != "guest:apply,guest:live-restore-on,docker:apply" || len(h.reports) != 2 {
|
||||
t.Fatalf("a BYO box got a host step: host=%+v calls=%s", ho, calls(w))
|
||||
}
|
||||
}
|
||||
@@ -223,7 +224,7 @@ func TestGuestFailure_SkipsTheHost(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{
|
||||
LayerGuest: {Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
g, ho := run2(l, "night")
|
||||
if g.Outcome != "refused" || ho.Outcome != "" || calls(w) != "guest:apply" {
|
||||
t.Fatalf("g=%+v h=%+v calls=%s", g, ho, calls(w))
|
||||
}
|
||||
@@ -236,7 +237,7 @@ func TestHealth_FailsAfterTheWait(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}},
|
||||
healthSeq: map[string][]*Health{LayerGuest: {bad, bad, bad, bad, bad, bad, bad, bad}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
g, ho := l.Run(context.Background(), 9201, "night")
|
||||
g, ho := run2(l, "night")
|
||||
if g.Outcome != "health_failed" || g.Healthy || !strings.Contains(g.HealthReason, "app was running") || ho.Outcome != "" {
|
||||
t.Fatalf("g=%+v h=%+v", g, ho)
|
||||
}
|
||||
@@ -251,7 +252,7 @@ func TestHealth_RecoversInsideTheWait(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting}},
|
||||
healthSeq: map[string][]*Health{LayerGuest: {starting}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "applied" || !g.Healthy {
|
||||
if g, _ := run2(l, "night"); g.Outcome != "applied" || !g.Healthy {
|
||||
t.Fatalf("g = %+v", g)
|
||||
}
|
||||
}
|
||||
@@ -261,7 +262,7 @@ func TestHost_TunnelDownFailsTheHostStep(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "openssl"}}}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Tunnel = fakeTunnel{hub.TunnelNotRunning}
|
||||
_, ho := l.Run(context.Background(), 9201, "night")
|
||||
_, ho := run2(l, "night")
|
||||
if ho.Outcome != "health_failed" || !strings.Contains(ho.HealthReason, "tunnel") {
|
||||
t.Fatalf("host = %+v", ho)
|
||||
}
|
||||
@@ -328,12 +329,12 @@ func TestHostHealthVerdict(t *testing.T) {
|
||||
func TestOncePerNight(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, pending: pend, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Run(context.Background(), 9201, "night")
|
||||
run2(l, "night")
|
||||
n := len(w.plans)
|
||||
if g, _ := l.Run(context.Background(), 9201, "night"); g.Outcome != "skipped" || len(w.plans) != n {
|
||||
if g, _ := run2(l, "night"); g.Outcome != "skipped" || len(w.plans) != n {
|
||||
t.Fatalf("a second night run in the same night ran: %+v", g)
|
||||
}
|
||||
if g, _ := l.Run(context.Background(), 9201, "debug"); g.Outcome == "skipped" {
|
||||
if g, _ := run2(l, "debug"); g.Outcome == "skipped" {
|
||||
t.Fatal("the debug action must not be throttled")
|
||||
}
|
||||
}
|
||||
@@ -344,13 +345,16 @@ func TestWrapperSuite(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Skip("python3 not available")
|
||||
}
|
||||
cmd := exec.Command(py, "-B", "../../configs/test_felhom_os_apply.py")
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("wrapper suite failed: %v\n%s", err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "OK") {
|
||||
t.Fatalf("wrapper suite did not report OK:\n%s", out)
|
||||
// the OS wrapper and (agent v0.142.0) the crash guard — both root programs in configs/ with their own suites
|
||||
for _, suite := range []string{"../../configs/test_felhom_os_apply.py", "../../configs/test_felhom_crash_guard.py"} {
|
||||
cmd := exec.Command(py, "-B", suite)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("%s failed: %v\n%s", suite, err, out)
|
||||
}
|
||||
if !strings.Contains(string(out), "OK") {
|
||||
t.Fatalf("%s did not report OK:\n%s", suite, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,8 +366,102 @@ func TestHostReport_CarriesRebootScanned(t *testing.T) {
|
||||
LayerHost: {RebootScanned: true, RebootNeeded: true, RestartNeeded: []string{"lxc-start"}},
|
||||
}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Run(context.Background(), 9201, "night")
|
||||
if len(h.reports) != 2 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
|
||||
run2(l, "night")
|
||||
if len(h.reports) != 3 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
|
||||
t.Fatalf("hub got %+v", h.reports)
|
||||
}
|
||||
}
|
||||
|
||||
// run2 is the guest + host reports of one pass (the tests written before the docker step).
|
||||
func run2(l *Leg, trigger string) (Report, Report) {
|
||||
p := l.Run(context.Background(), 9201, trigger)
|
||||
return p.Guest, p.Host
|
||||
}
|
||||
|
||||
// ---- the Docker step (`11` §5.8, agent v0.142.0) ----
|
||||
|
||||
// Ring 1 never takes an engine step in the night leg — only inside a signed operator job. Red-proof: drop the
|
||||
// `blk.Ring != 0` case in Run and the ring-1 pass makes a docker call.
|
||||
func TestDocker_Ring1NightLegNeverSteps(t *testing.T) {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") || strings.Contains(calls(w), "live-restore") {
|
||||
t.Fatalf("ring 1 took a docker step: %s", calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// An unhealthy earlier step skips the docker step.
|
||||
func TestDocker_SkippedAfterAnUnhealthyStep(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6"}}}},
|
||||
healthSeq: map[string][]*Health{}}
|
||||
bad := guestOK()
|
||||
bad.Controller = "unhealthy"
|
||||
w.applyRep[LayerGuest] = WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad}
|
||||
w.healthSeq[LayerGuest] = []*Health{bad, bad, bad, bad, bad, bad, bad, bad}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
if p.Docker.Layer != "" || strings.Contains(calls(w), "docker") {
|
||||
t.Fatalf("docker step ran after an unhealthy guest step: %s", calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// The docker plan is the slow lane, pending-docker for ring 0; the report carries only the engine set.
|
||||
func TestDocker_Ring0PlanAndReport(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
|
||||
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie"}},
|
||||
Installed: []Package{{Name: "docker-ce", Version: "5:29.8.2-1~debian.13~trixie", Origin: "Docker"}, {Name: "libc6", Version: "u4", Origin: "Debian"}},
|
||||
DockerEngine: "29.8.2", Authority: "ring0"}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
dp := w.plans[len(w.plans)-1]
|
||||
if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["select"] != "pending-docker" {
|
||||
t.Fatalf("docker plan = %v", dp)
|
||||
}
|
||||
d := p.Docker
|
||||
if d.Outcome != "applied" || !d.Healthy || d.DockerEngine != "29.8.2" || len(d.Installed) != 1 || d.Installed[0].Name != "docker-ce" {
|
||||
t.Fatalf("docker report = %+v", d)
|
||||
}
|
||||
}
|
||||
|
||||
// THE docker health rule. Red-proof: drop the id comparison (or the engine check) in DockerHealthVerdict and a case fails.
|
||||
func TestDockerHealthVerdict(t *testing.T) {
|
||||
before := guestOK()
|
||||
before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
|
||||
"app": {State: "running", Health: "healthy", ID: "b"}}
|
||||
same := guestOK()
|
||||
same.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
|
||||
"app": {State: "running", Health: "healthy", ID: "b"}}
|
||||
moved := guestOK()
|
||||
moved.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"},
|
||||
"app": {State: "running", Health: "healthy", ID: "c"}}
|
||||
if ok, why := DockerHealthVerdict(before, same, "29.8.2", "29.8.2"); !ok {
|
||||
t.Fatalf("same ids, right engine: %s", why)
|
||||
}
|
||||
if ok, _ := DockerHealthVerdict(before, moved, "29.8.2", "29.8.2"); ok {
|
||||
t.Fatal("a changed container id passed — live-restore failed and the apps restarted")
|
||||
}
|
||||
if ok, _ := DockerHealthVerdict(before, same, "29.8.2", "29.7.2"); ok {
|
||||
t.Fatal("the engine did not move and the step passed")
|
||||
}
|
||||
if EngineOf("5:29.8.2-1~debian.13~trixie") != "29.8.2" {
|
||||
t.Fatalf("EngineOf = %q", EngineOf("5:29.8.2-1~debian.13~trixie"))
|
||||
}
|
||||
}
|
||||
|
||||
// A changed id after the step → health_failed (the consequence, not only the verdict).
|
||||
func TestDocker_ChangedIDIsHealthFailed(t *testing.T) {
|
||||
before := guestOK()
|
||||
before.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "a"}}
|
||||
after := guestOK()
|
||||
after.Containers = map[string]Container{"felhom-controller": {State: "running", Health: "healthy", ID: "z"}}
|
||||
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerDocker: {
|
||||
Upgraded: []Package{{Name: "docker-ce", Version: "5:29.8.2-1"}}, DockerEngine: "29.8.2",
|
||||
HealthBefore: before, HealthAfter: after}}, healthSeq: map[string][]*Health{}}
|
||||
w.healthSeq[LayerDocker] = []*Health{after, after, after, after, after, after, after, after}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
if p.Docker.Outcome != "health_failed" || !strings.Contains(p.Docker.HealthReason, "id changed") {
|
||||
t.Fatalf("docker = %+v", p.Docker)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user