OS updates, guest fast lane (11 §8 step 2): felhom-os-apply wrapper (R1-R13 refusals, repair first, snapshot.debian.org fallback), FELHOM_OSAPPLY sudoers, the OS leg after the primary backup, hub os_update block + os-report, --selftest=os-update

No automatic undo: a customer guest cannot be snapshotted (R-837, measured). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
2026-10-04 10:44:29 +02:00
parent 596238cc2e
commit 23a8ef3de4
13 changed files with 1877 additions and 1 deletions
@@ -0,0 +1,429 @@
+// Package osupdate is the agent's OS-update leg for the customer GUEST (`11-os-updates.md` §8 step 2, agent v0.140.0).
+//
+// It runs right after the night's successful whole-guest backup, while the backup goroutine still holds the host-wide
+// heavy-op gate (so it never overlaps a backup or a restore-test, `11` C10), at most once per night. All root work is
+// the wrapper `felhom-os-apply` (configs/, its own tests); this package only builds plans, calls the wrapper through
+// sudo, judges health and reports to the hub.
+//
+// NO AUTOMATIC UNDO in this release (R-837, measured 2026-10-04): a customer guest cannot be snapshotted — PVE
+// refuses any snapshot not named `vzdump` when the guest has host-path binds (mp8, mp9). A failed health check
+// therefore stops, reports `health_failed` and the hub mails the operator; the whole-guest backup taken minutes
+// earlier is the undo, by hand.
+package osupdate
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// WrapperPath is the pinned sudoers vector (configs/felhom-agent.sudoers FELHOM_OSAPPLY).
+const WrapperPath = "/usr/local/sbin/felhom-os-apply"
+
+// DefaultPlanDir is where plans are written (the sudoers glob names it).
+const DefaultPlanDir = "/var/lib/felhom-agent/os"
+
+// Package is one name=version with its origin.
+type Package struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+	Origin  string `json:"origin"`
+}
+
+// Pending is one update the guest's sources offer (origin as apt names it, possibly several).
+type Pending struct {
+	Name   string   `json:"name"`
+	From   string   `json:"from"`
+	To     string   `json:"to"`
+	Origin []string `json:"origin"`
+}
+
+// Container is one container's state as the wrapper saw it.
+type Container struct {
+	State  string `json:"state"`
+	Health string `json:"health"` // healthy | unhealthy | starting | none
+}
+
+// Health is the guest's health snapshot (the wrapper's `health` object).
+type Health struct {
+	DockerOK   bool                 `json:"docker_ok"`
+	NetworkOK  bool                 `json:"network_ok"`
+	Controller string               `json:"controller"`
+	Containers map[string]Container `json:"containers"`
+}
+
+// WrapperReport is the wrapper's OSAPPLY-REPORT object.
+type WrapperReport struct {
+	Mode                string          `json:"mode"`
+	Refused             json.RawMessage `json:"refused"`
+	Failed              json.RawMessage `json:"failed"`
+	Upgraded            []Package       `json:"upgraded"`
+	Installed           []Package       `json:"installed"`
+	Pending             []Pending       `json:"pending"`
+	RestartNeeded       []string        `json:"restart_needed"`
+	DockerRestartNeeded bool            `json:"docker_restart_needed"`
+	RebootNeeded        bool            `json:"reboot_needed"`
+	HealthBefore        *Health         `json:"health_before"`
+	HealthAfter         *Health         `json:"health_after"`
+	Health              *Health         `json:"health"`
+}
+
+func (w WrapperReport) refused() bool { return len(w.Refused) > 0 && string(w.Refused) != "null" }
+func (w WrapperReport) failed() bool  { return len(w.Failed) > 0 && string(w.Failed) != "null" }
+
+// Report is what the hub receives (hub osupdates.Report — field-exact).
+type Report struct {
+	RunID               string          `json:"run_id"`
+	Trigger             string          `json:"trigger"`
+	Mode                string          `json:"mode"`
+	Ring                int             `json:"ring"`
+	ReleaseID           string          `json:"release_id"`
+	Outcome             string          `json:"outcome"`
+	Healthy             bool            `json:"healthy"`
+	HealthReason        string          `json:"health_reason,omitempty"`
+	VMID                int             `json:"vmid"`
+	Upgraded            []Package       `json:"upgraded,omitempty"`
+	Installed           []Package       `json:"installed,omitempty"`
+	Pending             []Pending       `json:"pending,omitempty"`
+	NotCovered          []string        `json:"not_covered,omitempty"`
+	RestartNeeded       []string        `json:"restart_needed,omitempty"`
+	DockerRestartNeeded bool            `json:"docker_restart_needed,omitempty"`
+	RebootNeeded        bool            `json:"reboot_needed,omitempty"`
+	Refused             json.RawMessage `json:"refused,omitempty"`
+}
+
+// Reporter posts a report to the hub (*hub.Client).
+type Reporter interface {
+	PostOSReport(ctx context.Context, body []byte) error
+}
+
+// Leg runs one OS-update pass for the customer guest.
+type Leg struct {
+	Runner     proxmox.Runner
+	Hub        Reporter
+	Logger     *slog.Logger
+	PlanDir    string
+	StatePath  string        // last night run (once per night)
+	HealthWait time.Duration // how long health may take to come back (default 5 min)
+	HealthPoll time.Duration // default 15 s
+	MinGap     time.Duration // between night runs (default 20 h)
+	Now        func() time.Time
+	Sleep      func(context.Context, time.Duration)
+
+	mu    sync.Mutex
+	block *hub.WireOSUpdate
+	have  bool
+}
+
+// OnDesiredState stores the hub's os_update block (desired.RawConsumer — store only, never block).
+func (l *Leg) OnDesiredState(_ context.Context, resp *hub.DesiredStateResponse) {
+	if resp == nil {
+		return
+	}
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.block, l.have = resp.DesiredState.OSUpdate, true
+}
+
+// Block returns the newest os_update block. No block (an older hub, or nothing fetched yet) = ring 1, ON, no
+// release: the box reports and installs nothing.
+func (l *Leg) Block() hub.WireOSUpdate {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	if l.block == nil {
+		return hub.WireOSUpdate{Ring: 1, Enabled: true}
+	}
+	return *l.block
+}
+
+// SetBlock sets the block directly (the selftest fetches the desired state itself).
+func (l *Leg) SetBlock(b *hub.WireOSUpdate) {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+	l.block, l.have = b, true
+}
+
+func (l *Leg) now() time.Time {
+	if l.Now != nil {
+		return l.Now()
+	}
+	return time.Now()
+}
+
+func (l *Leg) log() *slog.Logger {
+	if l.Logger != nil {
+		return l.Logger
+	}
+	return slog.Default()
+}
+
+func (l *Leg) sleep(ctx context.Context, d time.Duration) {
+	if l.Sleep != nil {
+		l.Sleep(ctx, d)
+		return
+	}
+	select {
+	case <-ctx.Done():
+	case <-time.After(d):
+	}
+}
+
+// IsFast reports whether every origin apt names is Debian / Debian-Security (the fast lane, `11` C3).
+func IsFast(origins []string) bool {
+	if len(origins) == 0 {
+		return false
+	}
+	for _, o := range origins {
+		if o != "Debian" && o != "Debian-Security" {
+			return false
+		}
+	}
+	return true
+}
+
+func fastOrigin(origins []string) string {
+	for _, o := range origins {
+		if o == "Debian-Security" {
+			return o
+		}
+	}
+	return "Debian"
+}
+
+// HealthVerdict is THE health rule (`11` §5.4.1; pinned by TestHealthVerdict_*): after the run, docker answers, the
+// guest's network resolves, the controller's own health check is `healthy`, and every container that was running
+// before is running again — and healthy again if it was healthy before. "starting" is not yet healthy.
+func HealthVerdict(before, after *Health) (bool, string) {
+	if after == nil {
+		return false, "no health reading"
+	}
+	if !after.DockerOK {
+		return false, "docker does not answer"
+	}
+	if !after.NetworkOK {
+		return false, "the guest cannot resolve deb.debian.org"
+	}
+	if after.Controller != "healthy" {
+		return false, "the controller is " + after.Controller
+	}
+	if before == nil {
+		return true, ""
+	}
+	names := make([]string, 0, len(before.Containers))
+	for n := range before.Containers {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+	for _, n := range names {
+		b := before.Containers[n]
+		if b.State != "running" {
+			continue
+		}
+		a, ok := after.Containers[n]
+		if !ok || a.State != "running" {
+			return false, n + " was running and is not"
+		}
+		if b.Health == "healthy" && a.Health != "healthy" {
+			return false, n + " was healthy and is " + a.Health
+		}
+	}
+	return true, ""
+}
+
+// call writes the plan and runs the wrapper once.
+func (l *Leg) call(ctx context.Context, runID, mode string, vmid int, rel hub.WireOSRelease, pkgs []Package) (WrapperReport, error) {
+	dir := l.PlanDir
+	if dir == "" {
+		dir = DefaultPlanDir
+	}
+	if err := os.MkdirAll(dir, 0o700); err != nil {
+		return WrapperReport{}, fmt.Errorf("osupdate: plan dir: %w", err)
+	}
+	plan := map[string]any{"release_id": rel.ID, "layer": "guest", "lane": "fast", "vmid": vmid, "mode": mode,
+		"snapshot": rel.Snapshot, "packages": pkgs}
+	if pkgs == nil {
+		plan["packages"] = []Package{}
+	}
+	b, _ := json.Marshal(plan)
+	path := filepath.Join(dir, "plan-"+runID+"-"+mode+".json")
+	if err := os.WriteFile(path, b, 0o600); err != nil {
+		return WrapperReport{}, fmt.Errorf("osupdate: write plan: %w", err)
+	}
+	defer os.Remove(path)
+	stdout, stderr, err := l.Runner.Run(ctx, WrapperPath, "--plan", path)
+	for _, line := range strings.Split(strings.TrimSpace(string(stderr)), "\n") {
+		if strings.HasPrefix(line, "os-apply: ") {
+			l.log().Info("osupdate: wrapper", "line", line)
+		}
+	}
+	var rep WrapperReport
+	found := false
+	for _, line := range strings.Split(string(stdout), "\n") {
+		if strings.HasPrefix(line, "OSAPPLY-REPORT ") {
+			if jerr := json.Unmarshal([]byte(strings.TrimPrefix(line, "OSAPPLY-REPORT ")), &rep); jerr == nil {
+				found = true
+			}
+		}
+	}
+	if !found {
+		return rep, fmt.Errorf("osupdate: wrapper gave no report (err %v): %s", err, strings.TrimSpace(string(stderr)))
+	}
+	return rep, nil // a refusal / failure is IN the report (exit 2 / 3), not an error here
+}
+
+// Run is one pass: inventory → (switch, ring, plan) → apply → health → report. trigger is "night" or "debug".
+func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Report {
+	runID := l.now().UTC().Format("20060102T150405Z")
+	blk := l.Block()
+	rel := hub.WireOSRelease{ID: "ring0-" + runID}
+	if blk.Ring == 1 {
+		rel = hub.WireOSRelease{}
+		if blk.Release != nil {
+			rel = *blk.Release
+		}
+	}
+	rep := Report{RunID: runID, Trigger: trigger, Ring: blk.Ring, VMID: vmid, ReleaseID: rel.ID, Mode: "inventory"}
+	lg := l.log().With("run", runID, "vmid", vmid, "ring", blk.Ring, "trigger", trigger)
+
+	if trigger == "night" && l.StatePath != "" {
+		gap := l.MinGap
+		if gap == 0 {
+			gap = 20 * time.Hour
+		}
+		if b, err := os.ReadFile(l.StatePath); err == nil {
+			if last, perr := time.Parse(time.RFC3339, strings.TrimSpace(string(b))); perr == nil && l.now().Sub(last) < gap {
+				lg.Info("osupdate: skipped — already ran tonight", "last", last.UTC().Format(time.RFC3339))
+				rep.Outcome = "skipped"
+				return rep
+			}
+		}
+	}
+	lg.Info("osupdate: START", "enabled", blk.Enabled, "release", rel.ID)
+
+	inv, err := l.call(ctx, runID, "inventory", vmid, rel, nil)
+	if err != nil || inv.refused() || inv.failed() {
+		rep.Outcome, rep.Refused = "refused", inv.Refused
+		if err != nil {
+			rep.Outcome, rep.HealthReason = "failed", err.Error()
+		}
+		return l.finish(ctx, lg, rep)
+	}
+	var plan []Package
+	switch {
+	case !blk.Enabled:
+		rep.Outcome = "inventory"
+		lg.Info("osupdate: switched OFF for this box — reporting only")
+	case blk.Ring == 0:
+		for _, p := range inv.Pending {
+			if IsFast(p.Origin) {
+				plan = append(plan, Package{Name: p.Name, Version: p.To, Origin: fastOrigin(p.Origin)})
+			}
+		}
+	default:
+		for _, p := range rel.Packages {
+			plan = append(plan, Package{Name: p.Name, Version: p.Version, Origin: p.Origin})
+		}
+	}
+	pkgNames := map[string]bool{}
+	for _, p := range plan {
+		pkgNames[p.Name] = true
+	}
+	if blk.Enabled && len(plan) == 0 {
+		rep.Outcome = "nothing"
+	}
+	final := inv
+	if blk.Enabled && len(plan) > 0 {
+		rep.Mode = "apply"
+		ap, err := l.call(ctx, runID, "apply", vmid, rel, plan)
+		switch {
+		case err != nil:
+			rep.Outcome, rep.HealthReason = "failed", err.Error()
+			return l.finish(ctx, lg, rep)
+		case ap.refused():
+			rep.Outcome, rep.Refused = "refused", ap.Refused
+			return l.finish(ctx, lg, rep)
+		case ap.failed():
+			rep.Outcome, rep.Refused = "failed", ap.Failed
+		}
+		final = ap
+		rep.Upgraded = ap.Upgraded
+		if rep.Outcome == "" {
+			if len(ap.Upgraded) == 0 {
+				rep.Outcome = "nothing"
+			} else {
+				rep.Outcome = "applied"
+			}
+		}
+		// Health: compare with what the guest looked like BEFORE the run; give restarted services time.
+		wait, poll := l.HealthWait, l.HealthPoll
+		if wait == 0 {
+			wait = 5 * time.Minute
+		}
+		if poll == 0 {
+			poll = 15 * time.Second
+		}
+		deadline := l.now().Add(wait)
+		cur := ap.HealthAfter
+		for {
+			ok, why := HealthVerdict(ap.HealthBefore, cur)
+			rep.Healthy, rep.HealthReason = ok, why
+			if ok || !l.now().Before(deadline) || ctx.Err() != nil {
+				break
+			}
+			l.sleep(ctx, poll)
+			hr, herr := l.call(ctx, runID, "health", vmid, rel, nil)
+			if herr == nil && hr.Health != nil {
+				cur = hr.Health
+			}
+		}
+		if !rep.Healthy && rep.Outcome == "applied" {
+			rep.Outcome = "health_failed"
+		}
+	} else {
+		ok, why := HealthVerdict(nil, inv.HealthAfter)
+		rep.Healthy, rep.HealthReason = ok, why
+	}
+	rep.Installed, rep.Pending = final.Installed, final.Pending
+	rep.RestartNeeded, rep.DockerRestartNeeded, rep.RebootNeeded = final.RestartNeeded, final.DockerRestartNeeded, final.RebootNeeded
+	rep.NotCovered = notCovered(final.Pending, blk.Ring, pkgNames)
+	if trigger == "night" && l.StatePath != "" {
+		_ = os.WriteFile(l.StatePath, []byte(l.now().UTC().Format(time.RFC3339)), 0o600)
+	}
+	return l.finish(ctx, lg, rep)
+}
+
+// notCovered lists pending updates no approved release covers: in ring 0 everything outside the fast lane; in
+// ring 1 also every fast-lane update the release did not name.
+func notCovered(pending []Pending, ring int, planned map[string]bool) []string {
+	var out []string
+	for _, p := range pending {
+		if !IsFast(p.Origin) || (ring == 1 && !planned[p.Name]) {
+			out = append(out, p.Name)
+		}
+	}
+	return out
+}
+
+func (l *Leg) finish(ctx context.Context, lg *slog.Logger, rep Report) Report {
+	lg.Info("osupdate: DONE", "outcome", rep.Outcome, "healthy", rep.Healthy, "reason", rep.HealthReason,
+		"upgraded", len(rep.Upgraded), "pending", len(rep.Pending), "not_covered", len(rep.NotCovered), "restart_needed", len(rep.RestartNeeded))
+	if l.Hub != nil {
+		body, _ := json.Marshal(rep)
+		rctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), time.Minute)
+		defer cancel()
+		if err := l.Hub.PostOSReport(rctx, body); err != nil {
+			lg.Warn("osupdate: reporting to the hub failed (the run itself is done)", "err", err)
+		}
+	}
+	return rep
+}
@@ -0,0 +1,265 @@
+package osupdate
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
+)
+
+// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per mode.
+type fakeWrapper struct {
+	t         *testing.T
+	pending   []Pending
+	applyRep  WrapperReport
+	healthSeq []*Health // answers to successive "health" calls
+	plans     []map[string]any
+}
+
+func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
+	if name != WrapperPath || len(args) != 2 || args[0] != "--plan" {
+		f.t.Fatalf("unexpected command %s %v", name, args)
+	}
+	b, err := os.ReadFile(args[1])
+	if err != nil {
+		f.t.Fatal(err)
+	}
+	var plan map[string]any
+	json.Unmarshal(b, &plan)
+	f.plans = append(f.plans, plan)
+	var rep WrapperReport
+	healthy := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
+		"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "running", Health: "healthy"}}}
+	switch plan["mode"] {
+	case "inventory":
+		rep = WrapperReport{Mode: "inventory", Pending: f.pending, HealthAfter: healthy,
+			Installed: []Package{{Name: "libc6", Version: "u3", Origin: "Debian"}}}
+	case "apply":
+		rep = f.applyRep
+		rep.Mode = "apply"
+		if rep.HealthBefore == nil {
+			rep.HealthBefore = healthy
+		}
+		if rep.HealthAfter == nil {
+			rep.HealthAfter = healthy
+		}
+	case "health":
+		if len(f.healthSeq) > 0 {
+			rep.Health, f.healthSeq = f.healthSeq[0], f.healthSeq[1:]
+		} else {
+			rep.Health = healthy
+		}
+	}
+	out, _ := json.Marshal(rep)
+	return []byte("OSAPPLY-REPORT " + string(out) + "\n"), []byte("os-apply: DONE rc=0\n"), nil
+}
+
+func (f *fakeWrapper) RunStdin(ctx context.Context, _ io.Reader, name string, args ...string) ([]byte, []byte, error) {
+	return f.Run(ctx, name, args...)
+}
+
+type fakeHub struct{ reports []Report }
+
+func (h *fakeHub) PostOSReport(_ context.Context, body []byte) error {
+	var r Report
+	json.Unmarshal(body, &r)
+	h.reports = append(h.reports, r)
+	return nil
+}
+
+func newLeg(t *testing.T, w *fakeWrapper, blk *hub.WireOSUpdate) (*Leg, *fakeHub) {
+	h := &fakeHub{}
+	now := time.Date(2026, 10, 4, 4, 0, 0, 0, time.UTC)
+	l := &Leg{Runner: w, Hub: h, PlanDir: t.TempDir(), StatePath: filepath.Join(t.TempDir(), "last"),
+		HealthWait: time.Minute, HealthPoll: 10 * time.Second,
+		Now:   func() time.Time { return now },
+		Sleep: func(_ context.Context, d time.Duration) { now = now.Add(d) }}
+	if blk != nil {
+		l.SetBlock(blk)
+	}
+	return l, h
+}
+
+var pend = []Pending{
+	{Name: "libc6", From: "u3", To: "u4", Origin: []string{"Debian"}},
+	{Name: "openssl", From: "u1", To: "u3", Origin: []string{"Debian-Security", "Debian"}},
+	{Name: "docker-ce", From: "29.7", To: "29.8", Origin: []string{"Docker CE"}},
+}
+
+func modes(w *fakeWrapper) string {
+	var m []string
+	for _, p := range w.plans {
+		m = append(m, p["mode"].(string))
+	}
+	return strings.Join(m, ",")
+}
+
+// Ring 0 plans every pending FAST-LANE update (Docker excluded, `11` C3) and reports what it now runs.
+func TestRing0_PlansTheFastLaneOnly(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4"}, {Name: "openssl", Version: "u3"}}, Pending: pend[2:]}}
+	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	rep := l.Run(context.Background(), 9201, "night")
+	if rep.Outcome != "applied" || !rep.Healthy {
+		t.Fatalf("rep = %+v", rep)
+	}
+	pk := w.plans[1]["packages"].([]any)
+	if len(pk) != 2 {
+		t.Fatalf("ring-0 plan = %v, want libc6 + openssl only", pk)
+	}
+	if o := pk[1].(map[string]any)["origin"]; o != "Debian-Security" {
+		t.Fatalf("openssl origin = %v", o)
+	}
+	if w.plans[1]["snapshot"] != "" {
+		t.Fatalf("ring 0 installs from live sources, snapshot = %v", w.plans[1]["snapshot"])
+	}
+	if len(rep.NotCovered) != 1 || rep.NotCovered[0] != "docker-ce" {
+		t.Fatalf("not covered = %v", rep.NotCovered)
+	}
+	if len(h.reports) != 1 || h.reports[0].Outcome != "applied" {
+		t.Fatalf("hub got %+v", h.reports)
+	}
+}
+
+// Ring 1 installs EXACTLY the approved release (its versions, its snapshot), nothing it computed itself.
+func TestRing1_InstallsExactlyTheRelease(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6", Version: "u4-approved"}}, Pending: pend[1:]}}
+	rel := &hub.WireOSRelease{ID: "os-1", Snapshot: "20261004T080000Z", Packages: []hub.WireOSPackage{{Name: "libc6", Version: "u4-approved", Origin: "Debian"}}}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true, Release: rel})
+	rep := l.Run(context.Background(), 9201, "night")
+	if rep.Outcome != "applied" || rep.ReleaseID != "os-1" {
+		t.Fatalf("rep = %+v", rep)
+	}
+	ap := w.plans[1]
+	pk := ap["packages"].([]any)
+	if len(pk) != 1 || pk[0].(map[string]any)["version"] != "u4-approved" || ap["snapshot"] != "20261004T080000Z" || ap["release_id"] != "os-1" {
+		t.Fatalf("ring-1 plan = %v", ap)
+	}
+	// openssl is pending and fast-lane but NOT in the release → not covered; docker-ce is never covered.
+	if strings.Join(rep.NotCovered, ",") != "openssl,docker-ce" {
+		t.Fatalf("not covered = %v", rep.NotCovered)
+	}
+}
+
+func TestRing1_NoReleaseInstallsNothing(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
+	rep := l.Run(context.Background(), 9201, "night")
+	if rep.Outcome != "nothing" || modes(w) != "inventory" {
+		t.Fatalf("rep=%+v modes=%s", rep, modes(w))
+	}
+}
+
+// No block from the hub (an older hub): ring 1, ON, no release → reports, installs nothing.
+func TestNoBlock_IsRing1Nothing(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend}
+	l, _ := newLeg(t, w, nil)
+	if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "nothing" || rep.Ring != 1 || modes(w) != "inventory" {
+		t.Fatalf("rep=%+v modes=%s", rep, modes(w))
+	}
+}
+
+// Switched OFF: the box reports but installs nothing (the brief, Part D 2).
+func TestSwitchOff_ReportsOnly(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend}
+	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: false})
+	rep := l.Run(context.Background(), 9201, "night")
+	if rep.Outcome != "inventory" || modes(w) != "inventory" || len(h.reports) != 1 || len(rep.Pending) != 3 {
+		t.Fatalf("rep=%+v modes=%s", rep, modes(w))
+	}
+}
+
+// Unhealthy after the run, and still unhealthy at the end of the wait → health_failed (the hub mails the operator).
+func TestHealth_FailsAfterTheWait(t *testing.T) {
+	bad := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{
+		"felhom-controller": {State: "running", Health: "healthy"}, "app": {State: "exited"}}}
+	w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: bad},
+		healthSeq: []*Health{bad, bad, bad, bad, bad, bad, bad, bad}}
+	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	rep := l.Run(context.Background(), 9201, "night")
+	if rep.Outcome != "health_failed" || rep.Healthy || !strings.Contains(rep.HealthReason, "app was running") {
+		t.Fatalf("rep = %+v", rep)
+	}
+	if h.reports[0].Outcome != "health_failed" {
+		t.Fatal("the hub was not told")
+	}
+}
+
+// A service that takes a moment to come back is not a failure: the poll sees it recover inside the wait.
+func TestHealth_RecoversInsideTheWait(t *testing.T) {
+	starting := &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}
+	w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}, HealthAfter: starting},
+		healthSeq: []*Health{starting}}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "applied" || !rep.Healthy {
+		t.Fatalf("rep = %+v", rep)
+	}
+}
+
+func TestHealthVerdict(t *testing.T) {
+	ok := &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}
+	cases := []struct {
+		name   string
+		before *Health
+		after  *Health
+		want   bool
+	}{
+		{"all good", ok, ok, true},
+		{"no reading", ok, nil, false},
+		{"docker down", ok, &Health{NetworkOK: true, Controller: "healthy"}, false},
+		{"no network", ok, &Health{DockerOK: true, Controller: "healthy"}, false},
+		{"controller starting", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "starting"}, false},
+		{"only the controller differs", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "unhealthy", Containers: map[string]Container{"a": {State: "running", Health: "healthy"}}}, false},
+		{"app gone", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{}}, false},
+		{"app unhealthy", ok, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy", Containers: map[string]Container{"a": {State: "running", Health: "unhealthy"}}}, false},
+		{"stopped before stays stopped", &Health{Containers: map[string]Container{"x": {State: "exited"}}}, &Health{DockerOK: true, NetworkOK: true, Controller: "healthy"}, true},
+	}
+	for _, c := range cases {
+		if got, why := HealthVerdict(c.before, c.after); got != c.want {
+			t.Errorf("%s: got %v (%s), want %v", c.name, got, why, c.want)
+		}
+	}
+}
+
+func TestOncePerNight(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Upgraded: []Package{{Name: "libc6"}}}}
+	l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	l.Run(context.Background(), 9201, "night")
+	n := len(w.plans)
+	if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "skipped" || len(w.plans) != n {
+		t.Fatalf("a second night run in the same night ran: %+v", rep)
+	}
+	if rep := l.Run(context.Background(), 9201, "debug"); rep.Outcome == "skipped" {
+		t.Fatal("the debug action must not be throttled")
+	}
+}
+
+func TestRefusedIsReported(t *testing.T) {
+	w := &fakeWrapper{t: t, pending: pend, applyRep: WrapperReport{Refused: json.RawMessage(`{"code":"R6","reason":"x"}`)}}
+	l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
+	if rep := l.Run(context.Background(), 9201, "night"); rep.Outcome != "refused" || h.reports[0].Outcome != "refused" {
+		t.Fatalf("rep = %+v", rep)
+	}
+}
+
+// The root wrapper's own suite (configs/test_felhom_os_apply.py) runs with `go test ./...` so CI covers it.
+func TestWrapperSuite(t *testing.T) {
+	py, err := exec.LookPath("python3")
+	if err != nil {
+		t.Skip("python3 not available")
+	}
+	cmd := exec.Command(py, "../../configs/test_felhom_os_apply.py")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("wrapper suite failed: %v\n%s", err, out)
+	}
+	if !strings.Contains(string(out), "OK") {
+		t.Fatalf("wrapper suite did not report OK:\n%s", out)
+	}
+}