v0.144.0 code: R8 measures the real download (R-865); an OS pass's report survives a killed agent (R-868); the debug pass runs from the saved block when the hub is away (R-866)
gates / gates (push) Successful in 19s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-05 07:14:32 +02:00
parent c8d12f1f2a
commit ca78c17b29
10 changed files with 662 additions and 28 deletions
+3
View File
@@ -73,6 +73,9 @@ func (e DockerStepExecutor) Execute(ctx context.Context, op string, params json.
// RunDockerSigned is one signed Docker step (ring 1 or an undo): live-restore first (decision 87, a no-op when on),
// then the docker layer with the signed envelope, which the wrapper verifies itself.
func (l *Leg) RunDockerSigned(ctx context.Context, vmid int, p DockerStepParams, blob []byte, sig string) Report {
unlock := l.lockPass(true)
defer unlock()
l.sendUnsentLocked(ctx) // R-868
runID := l.now().UTC().Format("20060102T150405Z")
lg := l.log().With("run", runID, "vmid", vmid, "trigger", "signed", "release", p.ReleaseID, "undo", p.Undo)
if err := l.EnsureLiveRestore(ctx, runID, vmid); err != nil {
+84 -16
View File
@@ -71,16 +71,16 @@ type Container struct {
// Health is one health reading. Guest layer: DockerOK..Containers. Host layer: HostServices, GuestRunning and the
// guest's own reading in Guest.
type Health struct {
DockerOK bool `json:"docker_ok"`
NetworkOK bool `json:"network_ok"`
Controller string `json:"controller"`
Containers map[string]Container `json:"containers"`
DockerOK bool `json:"docker_ok"`
NetworkOK bool `json:"network_ok"`
Controller string `json:"controller"`
Containers map[string]Container `json:"containers"`
// ControllerDockerOK: the controller reaches the engine from INSIDE its container (R-858, wrapper ≥ v0.142.1;
// nil from an older wrapper = not checked). Its own health check stayed "healthy" while it was blind.
ControllerDockerOK *bool `json:"controller_docker_ok,omitempty"`
HostServices map[string]string `json:"host_services,omitempty"`
GuestRunning *bool `json:"guest_running,omitempty"`
Guest *Health `json:"guest,omitempty"`
ControllerDockerOK *bool `json:"controller_docker_ok,omitempty"`
HostServices map[string]string `json:"host_services,omitempty"`
GuestRunning *bool `json:"guest_running,omitempty"`
Guest *Health `json:"guest,omitempty"`
}
// WrapperReport is the wrapper's OSAPPLY-REPORT object.
@@ -106,6 +106,13 @@ type WrapperReport struct {
LiveRestore json.RawMessage `json:"live_restore"`
Facts json.RawMessage `json:"facts"`
Bundle json.RawMessage `json:"bundle"` // the config bundle's result (R-840, mode "bundle")
// R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the
// agent process that started the pass. ReleaseID / VMID were always in the report.
RunID string `json:"run_id"`
Trigger string `json:"trigger"`
Ring *int `json:"ring"`
ReleaseID string `json:"release_id"`
VMID int `json:"vmid"`
}
func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
@@ -136,6 +143,8 @@ type Report struct {
DockerEngine string `json:"docker_engine,omitempty"` // docker layer: the engine after the step
Authority string `json:"authority,omitempty"` // docker layer: ring0 | signed
Undo bool `json:"undo,omitempty"` // docker layer: a signed undo (downgrade)
unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it
}
// Reporter posts a report to the hub (*hub.Client).
@@ -162,14 +171,65 @@ type Leg struct {
block *hub.WireOSUpdate
}
// planFile / reportFile: the plan the agent writes and the copy of the report the wrapper keeps beside it (R-868).
func planFile(dir, runID, layer, mode string) string {
return filepath.Join(dir, fmt.Sprintf("plan-%s-%s-%s.json", runID, layer, mode))
}
func reportFile(dir, runID, layer, mode string) string {
return filepath.Join(dir, fmt.Sprintf("report-%s-%s-%s.json", runID, layer, mode))
}
func (l *Leg) planDir() string {
if l.PlanDir == "" {
return DefaultPlanDir
}
return l.PlanDir
}
// OnDesiredState stores the hub's os_update block (desired.RawConsumer — store only, never block).
func (l *Leg) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) {
if resp == nil {
return
}
l.mu.Lock()
defer l.mu.Unlock()
l.block = resp.DesiredState.OSUpdate
l.mu.Unlock()
l.saveBlock(resp.DesiredState.OSUpdate)
}
// SavedBlockFile is the hub's newest os_update block as the daemon last received it (R-866, v0.144.0): the debug
// pass falls back to it when the hub cannot be reached, and says so.
const SavedBlockFile = "os-update-block.json"
type savedBlock struct {
SavedAt time.Time `json:"saved_at"`
Block *hub.WireOSUpdate `json:"block"`
}
func (l *Leg) saveBlock(b *hub.WireOSUpdate) {
dir := l.planDir()
if err := os.MkdirAll(dir, 0o700); err != nil {
return
}
body, _ := json.Marshal(savedBlock{SavedAt: l.now().UTC(), Block: b})
tmp := filepath.Join(dir, SavedBlockFile+".tmp")
if err := os.WriteFile(tmp, body, 0o600); err == nil {
_ = os.Rename(tmp, filepath.Join(dir, SavedBlockFile))
}
}
// LoadSavedBlock reads the block the daemon saved (R-866). ok=false: none saved yet.
func LoadSavedBlock(dir string) (b *hub.WireOSUpdate, savedAt time.Time, ok bool) {
raw, err := os.ReadFile(filepath.Join(dir, SavedBlockFile))
if err != nil {
return nil, time.Time{}, false
}
var s savedBlock
if json.Unmarshal(raw, &s) != nil {
return nil, time.Time{}, false
}
return s.Block, s.SavedAt, true
}
// Block returns the newest os_update block. No block (an older hub, or nothing fetched yet) = ring 1, ON, no
@@ -352,15 +412,12 @@ func DockerHealthVerdict(before, after *Health, wantEngine, gotEngine string) (b
// call writes the plan and runs the wrapper once.
func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (WrapperReport, error) {
dir := l.PlanDir
if dir == "" {
dir = DefaultPlanDir
}
dir := l.planDir()
if err := os.MkdirAll(dir, 0o700); err != nil {
return WrapperReport{}, fmt.Errorf("osupdate: plan dir: %w", err)
}
b, _ := json.Marshal(plan)
path := filepath.Join(dir, fmt.Sprintf("plan-%s-%s-%s.json", runID, plan["layer"], plan["mode"]))
path := planFile(dir, runID, fmt.Sprint(plan["layer"]), fmt.Sprint(plan["mode"]))
if err := os.WriteFile(path, b, 0o600); err != nil {
return WrapperReport{}, fmt.Errorf("osupdate: write plan: %w", err)
}
@@ -394,6 +451,9 @@ type Pass struct {
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
// only, after good earlier steps) the Docker engine set. trigger is "night" or "debug".
func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
unlock := l.lockPass(true)
defer unlock()
l.sendUnsentLocked(ctx) // R-868: a report a killed agent never sent goes first
g, h := l.runFast(ctx, vmid, trigger)
p := Pass{Guest: g, Host: h}
if g.Outcome == "skipped" {
@@ -521,7 +581,8 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
plan := map[string]any{"release_id": rel.ID, "layer": layer, "lane": lane, "vmid": vmid, "snapshot": rel.Snapshot,
"packages": []Package{}, "mode": "apply", "select": "listed"}
"packages": []Package{}, "mode": "apply", "select": "listed",
"run_id": runID, "trigger": trigger, "ring": blk.Ring} // R-868: echoed into the wrapper's kept copy
if rel.ID == "" {
plan["release_id"] = "none"
}
@@ -554,6 +615,9 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
}
rep.Mode = plan["mode"].(string)
wr, err := l.call(ctx, runID, plan)
if rep.Mode == "apply" {
rep.unsent = reportFile(l.planDir(), runID, layer, rep.Mode) // the wrapper kept a copy (R-868)
}
switch {
case err != nil:
rep.Outcome, rep.HealthReason = "failed", err.Error()
@@ -684,7 +748,11 @@ func (l *Leg) finish(ctx context.Context, lg *slog.Logger, rep Report) Report {
rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
defer cancel()
if err := l.Hub.PostOSReport(rctx, body); err != nil {
lg.Warn("osupdate: reporting to the hub failed (the run itself is done)", "err", err)
lg.Warn("osupdate: reporting to the hub failed (the run itself is done; the kept copy is sent at the next start or pass)", "err", err)
} else if rep.unsent != "" {
if rerr := os.Remove(rep.unsent); rerr != nil && !os.IsNotExist(rerr) {
lg.Warn("osupdate: could not delete the sent report's kept copy (it may be sent twice)", "path", rep.unsent, "err", rerr)
}
}
}
return rep
+18
View File
@@ -3,6 +3,7 @@ package osupdate
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"os/exec"
@@ -21,6 +22,7 @@ type fakeWrapper struct {
applyRep map[string]WrapperReport // per layer
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
plans []map[string]any
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
}
func yes() *bool { b := true; return &b }
@@ -72,6 +74,22 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
rep.Health = ok
}
}
if f.keep && plan["mode"] == "apply" {
kept := rep
kept.Layer, kept.RunID, _ = layer, fmt.Sprint(plan["run_id"]), 0
if tr, ok := plan["trigger"].(string); ok {
kept.Trigger = tr
}
if r, ok := plan["ring"].(float64); ok {
ri := int(r)
kept.Ring = &ri
}
kb, _ := json.Marshal(kept)
dst := filepath.Join(filepath.Dir(args[1]), "report-"+strings.TrimPrefix(filepath.Base(args[1]), "plan-"))
if err := os.WriteFile(dst, kb, 0o600); err != nil {
f.t.Fatal(err)
}
}
out, _ := json.Marshal(rep)
return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil
}
+178
View File
@@ -0,0 +1,178 @@
package osupdate
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"syscall"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)
// ── R-868 (v0.144.0): a pass whose agent was killed still reports ─────────────────────────────────────
//
// MEASURED 2026-10-05 02:57 UTC on demo-hp (night drill A5): the debug pass and the agent daemon were kill -9-ed while
// apt-get ran. The root wrapper (its own process under sudo) finished all six packages, but the agent that would have
// read its stdout and posted the report was gone; the hub never learned what the pass installed.
//
// THE MECHANISM: the wrapper writes its apply report to <plan dir>/report-<run>-<layer>-apply.json BEFORE printing
// it (configs/felhom-os-apply save_report). The agent deletes that copy once the hub has the report (finish). A copy
// still on disk is a report nobody sent: SendUnsent posts it — at the agent's start, and before every pass — and then
// deletes it. A pass lock (flock on <plan dir>/pass.lock, released by the kernel when a process dies) keeps the
// sender from picking up the copy of a pass that is still running, also across the daemon and a selftest process.
// Pinned by TestR868_* (unsent_test.go).
// lockPass takes the pass lock. block=false returns ok=false at once when another pass holds it. A lock that cannot
// be opened at all (no plan dir yet) does not stop a pass: the unlock is then a no-op.
func (l *Leg) lockPass(block bool) (unlock func()) {
u, _ := l.tryLockPass(block)
return u
}
func (l *Leg) tryLockPass(block bool) (unlock func(), ok bool) {
dir := l.planDir()
_ = os.MkdirAll(dir, 0o700)
f, err := os.OpenFile(filepath.Join(dir, "pass.lock"), os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
l.log().Warn("osupdate: pass lock unavailable — continuing without it", "err", err)
return func() {}, true
}
how := syscall.LOCK_EX
if !block {
how |= syscall.LOCK_NB
}
if err := syscall.Flock(int(f.Fd()), how); err != nil {
f.Close()
return func() {}, false
}
return func() { _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN); f.Close() }, true
}
// SendUnsent posts every report a pass kept on disk and nobody sent (R-868). It skips when a pass runs now (that
// pass sends them first). Called at the agent's start.
func (l *Leg) SendUnsent(ctx context.Context) int {
unlock, ok := l.tryLockPass(false)
if !ok {
l.log().Info("osupdate: a pass is running — its start sends any kept report")
return 0
}
defer unlock()
return l.sendUnsentLocked(ctx)
}
func (l *Leg) sendUnsentLocked(ctx context.Context) int {
if l.Hub == nil {
return 0 // nobody to send to: keep the copies for a process that has the hub
}
files, _ := filepath.Glob(filepath.Join(l.planDir(), "report-*.json"))
sent := 0
for _, f := range files {
b, err := os.ReadFile(f)
if err != nil {
l.log().Warn("osupdate: a kept report cannot be read", "path", f, "err", err)
continue
}
var wr WrapperReport
if err := json.Unmarshal(b, &wr); err != nil || wr.Layer == "" {
l.log().Warn("osupdate: a kept report is not a report — moved aside", "path", f, "err", err)
_ = os.Rename(f, f+".bad")
continue
}
rep := l.reportFromKept(ctx, wr, f)
lg := l.log().With("run", rep.RunID, "layer", rep.Layer, "vmid", rep.VMID, "ring", rep.Ring, "trigger", rep.Trigger)
lg.Info("osupdate: sending a report the agent never sent (the agent stopped mid-pass, R-868)", "path", f)
before := rep.unsent
_ = l.finish(ctx, lg, rep)
if _, err := os.Stat(before); os.IsNotExist(err) {
sent++
// the pass's plan file is left behind too when the agent was killed inside call()
_ = os.Remove(filepath.Join(filepath.Dir(f), "plan-"+strings.TrimPrefix(filepath.Base(f), "report-")))
}
}
return sent
}
// reportFromKept builds the hub report from a kept wrapper report, as runLayer would have. Health: the copy's own
// before/after reading; when that is not healthy after an install, one fresh reading (services restart after an
// install, and the pass that would have waited for them is gone).
func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string) Report {
ring := 1
if wr.Ring != nil {
ring = *wr.Ring
}
runID, trigger := wr.RunID, wr.Trigger
if runID == "" {
runID = strings.TrimSuffix(strings.TrimPrefix(filepath.Base(path), "report-"), ".json")
}
if trigger == "" {
trigger = "unknown"
}
rep := Report{RunID: runID, Layer: wr.Layer, Trigger: trigger, Mode: wr.Mode, Ring: ring, ReleaseID: wr.ReleaseID,
VMID: wr.VMID, unsent: path}
prefix := "sent after the agent stopped mid-pass (R-868)"
switch {
case wr.refused():
rep.Outcome, rep.Refused, rep.HealthReason = "refused", wr.Refused, prefix
return rep
case wr.failed():
rep.Outcome, rep.Refused = "failed", wr.Failed
case len(wr.Upgraded) == 0:
rep.Outcome = "nothing"
default:
rep.Outcome = "applied"
}
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
wantEngine := ""
for _, u := range wr.Upgraded {
if u.Name == "docker-ce" {
wantEngine = EngineOf(u.Version)
}
}
verdict := func(h *Health) (bool, string) {
switch wr.Layer {
case LayerDocker:
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
case LayerHost:
t := hub.TunnelUnknown
if l.Tunnel != nil {
t, _ = l.Tunnel.Status(ctx)
}
return HostHealthVerdict(wr.HealthBefore, h, t)
}
return HealthVerdict(wr.HealthBefore, h)
}
ok, why := verdict(wr.HealthAfter)
if !ok && len(wr.Upgraded) > 0 && wr.VMID > 0 {
lane := "fast"
if wr.Layer == LayerDocker {
lane = "slow"
}
if hr, err := l.call(ctx, "kept-"+runID, map[string]any{"release_id": "kept", "layer": wr.Layer, "lane": lane,
"vmid": wr.VMID, "mode": "health", "packages": []Package{}}); err == nil && hr.Health != nil {
ok, why = verdict(hr.Health)
}
}
rep.Healthy, rep.HealthReason = ok, prefix
if why != "" {
rep.HealthReason = prefix + ": " + why
}
if !ok && rep.Outcome == "applied" {
rep.Outcome = "health_failed"
}
rep.Installed, rep.Pending = wr.Installed, wr.Pending
rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = wr.RestartNeeded, wr.DockerRestartNeeded, wr.RebootNeeded
rep.RebootScanned = wr.RebootScanned
if wr.Layer == LayerDocker {
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
} else {
planned := map[string]bool{}
for _, u := range wr.Upgraded {
planned[u.Name] = true
}
rep.NotCovered = notCovered(wr.Pending, ring, planned)
}
return rep
}
+123
View File
@@ -0,0 +1,123 @@
package osupdate
import (
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"testing"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
)
// R-868 (v0.144.0). THE NIGHT'S SHAPE (A5, demo-hp 2026-10-05 02:57 UTC): the wrapper finished six packages, the agent
// was kill -9-ed before it read the report; the hub never got it. Here: the wrapper's kept copy and the plan file are
// on disk, a NEW agent process starts — it must send exactly one `applied` report and delete both files.
// COMPANION RED-PROOF: drop the SendUnsent body (return 0) → "the hub got no report".
func TestR868_KilledPassIsReportedAtStart(t *testing.T) {
w := &fakeWrapper{t: t}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
ring := 0
kept := WrapperReport{Mode: "apply", Layer: LayerGuest, RunID: "20261005T025700Z", Trigger: "debug", Ring: &ring, VMID: 9201,
ReleaseID: "ring0-20261005T025700Z", HealthBefore: guestOK(), HealthAfter: guestOK(),
Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}}
b, _ := json.Marshal(kept)
rp := reportFile(l.PlanDir, kept.RunID, LayerGuest, "apply")
pp := planFile(l.PlanDir, kept.RunID, LayerGuest, "apply")
must(t, os.WriteFile(rp, b, 0o600))
must(t, os.WriteFile(pp, []byte("{}"), 0o600))
if n := l.SendUnsent(context.Background()); n != 1 {
t.Fatalf("sent %d, want 1", n)
}
if len(h.reports) != 1 {
t.Fatalf("the hub got no report (or several): %+v", h.reports)
}
r := h.reports[0]
if r.Outcome != "applied" || !r.Healthy || r.Trigger != "debug" || r.RunID != kept.RunID || r.Ring != 0 || len(r.Upgraded) != 2 || r.VMID != 9201 {
t.Fatalf("report = %+v", r)
}
for _, p := range []string{rp, pp} {
if _, err := os.Stat(p); !os.IsNotExist(err) {
t.Fatalf("%s still on disk after the hub got it", filepath.Base(p))
}
}
// a second start sends nothing again — no duplicate report
if n := l.SendUnsent(context.Background()); n != 0 || len(h.reports) != 1 {
t.Fatalf("sent again: %d, reports %d", n, len(h.reports))
}
}
// A normal pass: the hub gets ONE report per layer and the kept copies are gone after it (nothing resent later).
// COMPANION RED-PROOF: drop the os.Remove(rep.unsent) in finish → the next pass resends → "2 guest reports".
func TestR868_NormalPassLeavesNoCopyAndNoDuplicate(t *testing.T) {
w := &fakeWrapper{t: t, keep: true, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "u4"}}}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Run(context.Background(), 9201, "debug")
left, _ := filepath.Glob(filepath.Join(l.PlanDir, "report-*.json"))
if len(left) != 0 {
t.Fatalf("kept copies left after the hub got them: %v", left)
}
l.Run(context.Background(), 9201, "debug") // the next pass sends kept copies first
guest := 0
for _, r := range h.reports {
if r.Layer == LayerGuest && r.Outcome == "applied" {
guest++
}
}
if guest != 2 {
t.Fatalf("%d guest reports for 2 passes (a duplicate or a loss)", guest)
}
}
type failingHub struct{ n int }
func (h *failingHub) PostOSReport(context.Context, []byte) error {
h.n++
return errors.New("hub away")
}
// The hub away: the copy stays, and goes at the next chance.
func TestR868_HubAwayKeepsTheCopy(t *testing.T) {
w := &fakeWrapper{t: t, keep: true, applyRep: map[string]WrapperReport{LayerGuest: {Upgraded: []Package{{Name: "libc6", Version: "u4"}}}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Appliance = false
l.Hub = &failingHub{}
l.Run(context.Background(), 9201, "night")
left, _ := filepath.Glob(filepath.Join(l.PlanDir, "report-*.json"))
if len(left) != 2 { // ring 0: the guest step and the Docker step each kept one
t.Fatalf("the copies must stay while the hub is away: %v", left)
}
h := &fakeHub{}
l.Hub = h
if n := l.SendUnsent(context.Background()); n != 2 {
t.Fatalf("sent %d: %+v", n, h.reports)
}
for _, r := range h.reports {
if r.Trigger != "night" || r.Ring != 0 {
t.Fatalf("the kept report lost its ids: %+v", r)
}
}
}
// A pass in progress holds the lock: the sender at start must not take that pass's copy (it would be sent twice).
func TestR868_RunningPassKeepsTheSenderOff(t *testing.T) {
w := &fakeWrapper{t: t}
l, h := newLeg(t, w, nil)
must(t, os.WriteFile(reportFile(l.PlanDir, "r1", LayerGuest, "apply"), []byte(`{"mode":"apply","layer":"guest"}`), 0o600))
unlock := l.lockPass(true)
if n := l.SendUnsent(context.Background()); n != 0 || len(h.reports) != 0 {
t.Fatalf("sent while a pass ran: %d", n)
}
unlock()
if n := l.SendUnsent(context.Background()); n != 1 {
t.Fatalf("not sent after the pass: %d", n)
}
}
func must(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}