R-812 option A: the Proxmox package lane (layer pve) + the /etc/pve write gate

The wrapper gains layer "pve" (slow lane): the host's Proxmox userspace
packages only — origin "Proxmox Debian Repository", never a kernel / boot /
firmware / microcode name (R14), no removal, no undo, a new package only from
an allow-list; authority = a signed os_pve_step or the root-owned ring-0 mark.
The night leg runs it in ring 0 after a healthy host step; ring 1 only by a
signed job (PVEStepExecutor). While it runs, the agent's own /etc/pve writes
(every non-GET API call, pct config verbs, pvesm, pveum, felhom-pbs-apply)
wait on internal/pvegate. Health = the host rule + unchanged container ids +
pveversion reads the installed pve-manager.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
2026-10-07 10:07:41 +02:00
parent ac90169a5d
commit ce1a4b4758
17 changed files with 990 additions and 41 deletions
+131 -6
View File
@@ -20,6 +20,7 @@ import (
"log/slog"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"sync"
@@ -27,6 +28,7 @@ import (
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
)
// WrapperPath is the pinned sudoers vector (configs/felhom-agent.sudoers FELHOM_OSAPPLY).
@@ -40,8 +42,21 @@ const (
LayerGuest = "guest"
LayerHost = "host"
LayerDocker = "docker" // the guest's Docker engine set — slow lane (`11` §5.8)
// LayerPVE is the HOST's Proxmox userspace packages — slow lane (R-812 option A, `09` §3 decision 163, `11` §5.10):
// ring 0 in the night leg after a healthy host step, ring 1 only inside a signed os_pve_step. Never a kernel.
LayerPVE = "pve"
)
// PVEOrigin is the origin apt prints for download.proxmox.com (the wrapper's PVE_ORIGIN); InstalledPVEOrigin is how
// the wrapper's inventory names the same source.
const (
PVEOrigin = "Proxmox Debian Repository"
InstalledPVEOrigin = "Proxmox"
)
// hostSlowRE mirrors the wrapper's HOST_SLOW_RE: kernel, boot, firmware and microcode names never ride the pve lane.
var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`)
// DockerNames are the six packages of the Docker engine set (the wrapper's DOCKER_NAMES).
var DockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
@@ -109,6 +124,8 @@ type WrapperReport struct {
// OOMCheck (R-528, `09` decision 157): the docker layer's memory-kill check, {result, oom_killed, oom_event,
// exit_code, image, detail}. Carried to the hub UNCHANGED; the agent never reads it.
OOMCheck json.RawMessage `json:"oom_check"`
// PVEManager: the pve layer — pveversion's pve-manager version after the step ("unknown" = unreadable).
PVEManager string `json:"pve_manager"`
// R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the
// agent process that started the pass. ReleaseID / VMID were always in the report.
RunID string `json:"run_id"`
@@ -149,6 +166,8 @@ type Report struct {
// OOMCheck: docker layer — the wrapper's oom_check object, byte-for-byte (R-528; the hub decides approval on it).
// Pinned by TestDocker_OOMCheckReachesTheHubUnchanged and TestR868_KeptCopyCarriesTheOOMCheck.
OOMCheck json.RawMessage `json:"oom_check,omitempty"`
// PVEManager: pve layer — pve-manager's version after the step (the hub's System page; R-812 option A).
PVEManager string `json:"pve_manager,omitempty"`
unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it
}
@@ -386,6 +405,36 @@ func EngineOf(pkgVersion string) string {
return v
}
// PVEHealthVerdict is THE Proxmox-package-step health rule (R-812 option A; pinned by TestPVEHealthVerdict): the host
// rule (every host service active, the guest running and healthy, the tunnel running), plus every container running at
// the start still runs as the SAME container (a Proxmox step must not restart the household's apps), plus pveversion
// now reports the pve-manager the step installed (wantPVE "" = pve-manager was not in the step).
func PVEHealthVerdict(before, after *Health, tunnel, wantPVE, gotPVE string) (bool, string) {
if ok, why := HostHealthVerdict(before, after, tunnel); !ok {
return false, why
}
if before != nil && before.Guest != nil && after.Guest != nil {
names := make([]string, 0, len(before.Guest.Containers))
for n := range before.Guest.Containers {
names = append(names, n)
}
sort.Strings(names)
for _, n := range names {
b := before.Guest.Containers[n]
if b.State != "running" || b.ID == "" {
continue
}
if a := after.Guest.Containers[n]; a.ID != b.ID {
return false, n + " is a new container (id changed) — the Proxmox step restarted the household's app"
}
}
}
if wantPVE != "" && gotPVE != wantPVE {
return false, "pveversion reads pve-manager " + gotPVE + ", not " + wantPVE
}
return true, ""
}
// DockerHealthVerdict is THE Docker-step health rule (`11` §5.8; pinned by TestDockerHealthVerdict): the guest rule,
// plus every container running at the start still runs as the SAME container (same id — a changed id means the
// household's apps restarted, which `live-restore` exists to prevent), plus the engine now reports the version the step
@@ -451,7 +500,7 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
// Pass is one leg's reports; an empty Layer means the step did not run.
type Pass struct {
Guest, Host, Docker Report
Guest, Host, Docker, PVE Report
}
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
@@ -479,13 +528,44 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
if err := l.EnsureLiveRestore(ctx, g.RunID, vmid); err != nil {
p.Docker = l.finish(ctx, lg, Report{RunID: g.RunID, Layer: LayerDocker, Trigger: trigger, Ring: 0, VMID: vmid,
Mode: "apply", Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
return p
} else {
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
}
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
}
// R-812 option A: the Proxmox package step — ring 0, an appliance, after a HEALTHY host step (it is a host change).
// A Docker step's outcome does not gate it (the Docker set lives in the guest). Pinned by TestPVE_*.
switch {
case blk.Ring != 0 || !blk.Enabled:
lg.Info("osupdate: pve step skipped — ring 1 takes a Proxmox set only inside a signed operator job (`11` §5.10)", "ring", blk.Ring, "enabled", blk.Enabled)
case !l.Appliance || h.Layer == "" || !okStep(h):
lg.Info("osupdate: pve step skipped — no healthy host step this pass (an appliance only)", "appliance", l.Appliance, "host_outcome", h.Outcome)
default:
p.PVE = l.runPVE(ctx, g.RunID, vmid, trigger, blk, dockerOpts{})
}
return p
}
// pveDrainWait bounds how long a pve step waits for the agent's own /etc/pve writes in flight (pvegate).
var pveDrainWait = 2 * time.Minute
// runPVE runs the pve layer while holding pvegate: the agent's own /etc/pve writes wait until it ends.
func (l *Leg) runPVE(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate, do dockerOpts) Report {
lg := l.log().With("run", runID, "layer", LayerPVE, "vmid", vmid, "trigger", trigger)
dctx, cancel := context.WithTimeout(ctx, pveDrainWait)
end, err := pvegate.Step(dctx)
cancel()
if err != nil {
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerPVE, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply",
ReleaseID: do.releaseID, Outcome: "failed", HealthReason: "an agent write to /etc/pve did not finish in time (pvegate): " + err.Error()})
}
lg.Info("osupdate: pve step holds the /etc/pve write gate — the agent's own writes wait until it ends")
defer func() {
end()
lg.Info("osupdate: pve step released the /etc/pve write gate")
}()
return l.runLayer(ctx, runID, LayerPVE, vmid, trigger, blk, do)
}
// dockerOpts is a signed Docker step (DockerStepExecutor); the zero value is ring 0's unsigned "pending-docker".
type dockerOpts struct {
releaseID string
@@ -571,7 +651,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
wire = blk.HostRelease
}
lane := "fast"
if layer == LayerDocker {
if layer == LayerDocker || layer == LayerPVE {
lane = "slow"
if do.signed != nil {
rel = hub.WireOSRelease{ID: do.releaseID}
@@ -604,6 +684,13 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
}
case layer == LayerDocker:
plan["select"] = "pending-docker" // ring 0: the wrapper checks the box's ROOT-OWNED ring-0 mark itself
case layer == LayerPVE && do.signed != nil:
plan["packages"], plan["signed"] = do.packages, do.signed
for _, p := range do.packages {
planned[p.Name] = true
}
case layer == LayerPVE:
plan["select"] = "pending-pve" // ring 0: the same root-owned mark; the wrapper picks installed Proxmox userspace
case !blk.Enabled:
plan["mode"] = "inventory"
lg.Info("osupdate: switched OFF for this box — reporting only")
@@ -637,6 +724,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
rep.OOMCheck = rawOrNil(wr.OOMCheck)
rep.PVEManager = wr.PVEManager
if rep.Outcome == "" {
switch {
case rep.Mode == "inventory" && !blk.Enabled:
@@ -654,21 +742,27 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
}
// Health: compare with the start of the pass; give restarted services time (only after an install).
cur := wr.HealthAfter
wantEngine := ""
wantEngine, wantPVE := "", ""
for _, u := range wr.Upgraded {
if u.Name == "docker-ce" {
wantEngine = EngineOf(u.Version)
}
if u.Name == "pve-manager" {
wantPVE = u.Version
}
}
verdict := func(h *Health) (bool, string) {
if layer == LayerDocker {
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
}
if layer == LayerHost {
if layer == LayerHost || layer == LayerPVE {
t := hub.TunnelUnknown
if l.Tunnel != nil {
t, _ = l.Tunnel.Status(ctx)
}
if layer == LayerPVE {
return PVEHealthVerdict(wr.HealthBefore, h, t, wantPVE, wr.PVEManager)
}
return HostHealthVerdict(wr.HealthBefore, h, t)
}
return HealthVerdict(wr.HealthBefore, h)
@@ -708,6 +802,10 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
// the docker report carries the engine set only (the guest report already carries the Debian packages)
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
rep.NotCovered = nil
} else if layer == LayerPVE {
// the pve report carries the Proxmox userspace set only — the hub's candidate is built from it
rep.Installed, rep.Pending = onlyPVE(wr.Installed), onlyPVEPending(wr.Pending)
rep.NotCovered = nil
} else {
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
}
@@ -732,6 +830,33 @@ func onlyDocker(in []Package) []Package {
return out
}
// onlyPVE keeps the installed Proxmox-origin packages the pve lane may touch (never a kernel / boot / firmware name).
func onlyPVE(in []Package) []Package {
var out []Package
for _, p := range in {
if (p.Origin == InstalledPVEOrigin || p.Origin == PVEOrigin) && !hostSlowRE.MatchString(p.Name) && !DockerNames[p.Name] {
out = append(out, p)
}
}
return out
}
func onlyPVEPending(in []Pending) []Pending {
var out []Pending
for _, p := range in {
if p.From == "" || hostSlowRE.MatchString(p.Name) || DockerNames[p.Name] {
continue
}
for _, o := range p.Origin {
if o == PVEOrigin {
out = append(out, p)
break
}
}
}
return out
}
func onlyDockerPending(in []Pending) []Pending {
var out []Pending
for _, p := range in {
+22 -12
View File
@@ -13,16 +13,18 @@ import (
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
)
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per layer and mode.
type fakeWrapper struct {
t *testing.T
pending []Pending
applyRep map[string]WrapperReport // per layer
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
plans []map[string]any
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
t *testing.T
pending []Pending
applyRep map[string]WrapperReport // per layer
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
plans []map[string]any
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
pveGateHeld bool
}
func yes() *bool { b := true; return &b }
@@ -50,9 +52,12 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
f.plans = append(f.plans, plan)
layer := plan["layer"].(string)
ok := guestOK()
if layer == LayerHost {
if layer == LayerHost || layer == LayerPVE {
ok = hostOK()
}
if layer == LayerPVE && plan["mode"] == "apply" {
f.pveGateHeld = pvegate.Stepping() // R-812: the /etc/pve write gate must be held while the pve step runs
}
var rep WrapperReport
switch plan["mode"] {
case "inventory":
@@ -160,8 +165,8 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
t.Fatalf("guest %+v\nhost %+v", g, ho)
}
if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply" {
t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore and the ring-0 docker step", calls(w))
if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply,pve:apply" {
t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore, the ring-0 docker step and the pve step", calls(w))
}
for _, p := range w.plans[:2] {
if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
@@ -171,7 +176,7 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
t.Fatalf("not covered = %v", g.NotCovered)
}
if len(h.reports) != 3 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker {
if len(h.reports) != 4 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker || h.reports[3].Layer != LayerPVE {
t.Fatalf("hub got %+v", h.reports)
}
}
@@ -389,7 +394,7 @@ func TestHostReport_CarriesRebootScanned(t *testing.T) {
}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
run2(l, "night")
if len(h.reports) != 3 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
if len(h.reports) != 4 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
t.Fatalf("hub got %+v", h.reports)
}
}
@@ -436,7 +441,12 @@ func TestDocker_Ring0PlanAndReport(t *testing.T) {
DockerEngine: "29.8.2", Authority: "ring0"}}}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
dp := w.plans[len(w.plans)-1]
var dp map[string]any
for _, x := range w.plans {
if x["layer"] == "docker" {
dp = x
}
}
if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["select"] != "pending-docker" {
t.Fatalf("docker plan = %v", dp)
}
+152
View File
@@ -0,0 +1,152 @@
package osupdate
import (
"context"
"encoding/base64"
"encoding/json"
"strings"
"testing"
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)
// ---- the Proxmox package step (R-812 option A, `09` §3 decision 163, `11` §5.10) ----
// Ring 0: after a healthy host step the leg runs the pve layer — slow lane, select pending-pve — while holding the
// /etc/pve write gate; the report carries only Proxmox userspace packages and pve-manager's version.
//
// COMPANION RED-PROOF (observed): call runLayer instead of runPVE in Run → "the /etc/pve write gate was not held".
func TestPVE_Ring0PlanGateAndReport(t *testing.T) {
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {
Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}},
Installed: []Package{{Name: "pve-manager", Version: "9.2.21", Origin: "Proxmox"},
{Name: "proxmox-kernel-helper", Version: "9.0.4", Origin: "Proxmox"}, {Name: "libc6", Version: "u4", Origin: "Debian"}},
Pending: []Pending{{Name: "qemu-server", From: "9.0.1", To: "9.0.9", Origin: []string{PVEOrigin}},
{Name: "proxmox-kernel-7.0", From: "7.0.2", To: "7.0.14", Origin: []string{PVEOrigin}},
{Name: "libc6", From: "u3", To: "u4", Origin: []string{"Debian"}}},
PVEManager: "9.2.21", Authority: "ring0"}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
var pp map[string]any
for _, x := range w.plans {
if x["layer"] == LayerPVE {
pp = x
}
}
if pp == nil || pp["lane"] != "slow" || pp["select"] != "pending-pve" {
t.Fatalf("pve plan = %v (calls %s)", pp, calls(w))
}
if !w.pveGateHeld {
t.Fatal("the /etc/pve write gate was not held while the pve step ran")
}
if pvegate.Stepping() {
t.Fatal("the gate must be released after the step")
}
r := p.PVE
if r.Outcome != "applied" || !r.Healthy || r.PVEManager != "9.2.21" {
t.Fatalf("pve report = %+v", r)
}
if len(r.Installed) != 1 || r.Installed[0].Name != "pve-manager" || len(r.Pending) != 1 || r.Pending[0].Name != "qemu-server" {
t.Fatalf("the pve report must carry Proxmox userspace only: installed=%v pending=%v", r.Installed, r.Pending)
}
if h.reports[len(h.reports)-1].Layer != LayerPVE {
t.Fatalf("the hub must get the pve report: %+v", h.reports)
}
}
// Ring 1 never takes a Proxmox step in the night leg.
func TestPVE_Ring1NightLegNeverSteps(t *testing.T) {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
if p := l.Run(context.Background(), 9201, "night"); p.PVE.Layer != "" || strings.Contains(calls(w), "pve") {
t.Fatalf("ring 1 took a pve step: %s", calls(w))
}
}
// No healthy host step (a BYO box, or an unhealthy host step) → no pve step.
func TestPVE_SkippedWithoutAHealthyHostStep(t *testing.T) {
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l.Appliance = false
if p := l.Run(context.Background(), 9201, "night"); p.PVE.Layer != "" || strings.Contains(calls(w), "pve") {
t.Fatalf("a BYO box took a pve step: %s", calls(w))
}
w2 := &fakeWrapper{t: t}
l2, _ := newLeg(t, w2, &hub.WireOSUpdate{Ring: 0, Enabled: true})
l2.Tunnel = fakeTunnel{"stopped"} // the host step reads unhealthy
w2.applyRep = map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "libc6", Version: "u4"}}}}
if p := l2.Run(context.Background(), 9201, "night"); p.PVE.Layer != "" || strings.Contains(calls(w2), "pve") {
t.Fatalf("a pve step ran after an unhealthy host step: %s", calls(w2))
}
}
// A write in flight that never finishes makes the pve step give up (failed), never run without the gate.
func TestPVE_GivesUpWhenAWriteDoesNotFinish(t *testing.T) {
old := pveDrainWait
pveDrainWait = 50_000_000 // 50 ms
defer func() { pveDrainWait = old }()
rel, _, _ := pvegate.Write(context.Background())
defer rel()
w := &fakeWrapper{t: t}
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
p := l.Run(context.Background(), 9201, "night")
if p.PVE.Outcome != "failed" || strings.Contains(calls(w), "pve") {
t.Fatalf("the pve step must fail without a wrapper call: %+v calls=%s", p.PVE, calls(w))
}
}
// THE pve health rule. COMPANION RED-PROOF (observed): drop the container-id loop or the pve-manager check in
// PVEHealthVerdict → the matching case below fails.
func TestPVEHealthVerdict(t *testing.T) {
before, after := hostOK(), hostOK()
before.Guest.Containers["app"] = Container{State: "running", Health: "healthy", ID: "a1"}
after.Guest.Containers["app"] = Container{State: "running", Health: "healthy", ID: "a1"}
if ok, why := PVEHealthVerdict(before, after, hub.TunnelRunning, "9.2.21", "9.2.21"); !ok {
t.Fatalf("healthy step read unhealthy: %s", why)
}
if ok, _ := PVEHealthVerdict(before, after, hub.TunnelRunning, "9.2.21", "9.2.2"); ok {
t.Fatal("pveversion still on the old pve-manager must fail")
}
after.Guest.Containers["app"] = Container{State: "running", Health: "healthy", ID: "b2"}
if ok, why := PVEHealthVerdict(before, after, hub.TunnelRunning, "", "9.2.2"); ok || !strings.Contains(why, "id changed") {
t.Fatalf("an app restarted by the Proxmox step must fail, got ok=%v %q", ok, why)
}
}
// The signed executor hands the RAW envelope and the exact list to the wrapper's pve layer.
func TestPVEStepExecutor_PassesTheSignedEnvelope(t *testing.T) {
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {
Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}}, PVEManager: "9.2.21", Authority: "signed"}}}
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
e := PVEStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
params, _ := json.Marshal(PVEStepParams{ReleaseID: "os-pve-1", Packages: []Package{{Name: "pve-manager", Version: "9.2.21", Origin: PVEOrigin}}})
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_pve_step"}`), Sig: []byte("SIG")})
if err := e.Execute(ctx, OpPVEStep, params); err != nil {
t.Fatal(err)
}
pp := w.plans[len(w.plans)-1]
sg, _ := pp["signed"].(map[string]any)
if pp["layer"] != LayerPVE || pp["lane"] != "slow" || pp["release_id"] != "os-pve-1" || sg == nil ||
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_pve_step"}`)) || sg["sig"] != "SIG" {
t.Fatalf("pve plan = %v", pp)
}
if !w.pveGateHeld || calls(w) != "pve:apply" || len(h.reports) != 1 || h.reports[0].Trigger != "signed" {
t.Fatalf("gate=%v calls=%s reports=%+v", w.pveGateHeld, calls(w), h.reports)
}
if err := e.Execute(context.Background(), OpPVEStep, params); err == nil {
t.Fatal("no envelope must refuse")
}
if err := e.Execute(context.Background(), OpDockerStep, params); err != signedjobs.ErrNoExecutor {
t.Fatalf("another op must pass through the chain: %v", err)
}
}
// os_pve_step is never benign.
func TestPVEStep_IsDestructiveClass(t *testing.T) {
if reconcile.Classify(reconcile.ClassOSPVEStep, reconcile.Provenance{}) != reconcile.Destructive {
t.Fatal("os_pve_step must be destructive-class (signed, operational key)")
}
}
+85
View File
@@ -0,0 +1,85 @@
package osupdate
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
)
// OpPVEStep is the signed op class of a Proxmox package step (R-812 option A, `11` §5.10): a ring-1 box takes an
// approved Proxmox set only through it. No undo in this release. CC may sign it until the first paying customer.
const OpPVEStep = "os_pve_step"
// PVEStepParams are the signed params. The wrapper compares Packages with the plan byte-for-byte.
type PVEStepParams struct {
ReleaseID string `json:"release_id"`
Packages []Package `json:"packages"`
VMID int `json:"vmid,omitempty"`
}
// PVEStepExecutor runs a verified os_pve_step (signedjobs.Executor) under the host-wide heavy-op gate (Gate) and the
// /etc/pve write gate (inside runPVE).
type PVEStepExecutor struct {
Leg *Leg
Guest func(ctx context.Context) (int, error)
Gate func(ctx context.Context) (release func(), err error)
}
// Execute implements signedjobs.Executor.
func (e PVEStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
if op != OpPVEStep {
return signedjobs.ErrNoExecutor
}
so, ok := signedjobs.SignedOpFrom(ctx)
if !ok {
return fmt.Errorf("os_pve_step: no signed envelope in the context — the wrapper could not verify it")
}
var p PVEStepParams
if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 {
return fmt.Errorf("os_pve_step: params must name the Proxmox set: %v", err)
}
vmid := p.VMID
if vmid == 0 {
if e.Guest == nil {
return fmt.Errorf("os_pve_step: no vmid and no guest finder")
}
v, err := e.Guest(ctx)
if err != nil {
return fmt.Errorf("os_pve_step: find the customer guest: %w", err)
}
vmid = v
}
if e.Gate != nil {
release, err := e.Gate(ctx)
if err != nil {
return fmt.Errorf("os_pve_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
}
defer release()
}
rep := e.Leg.RunPVESigned(ctx, vmid, p, so.Blob, string(so.Sig))
switch rep.Outcome {
case "applied", "nothing":
if rep.Healthy {
return nil
}
}
return fmt.Errorf("os_pve_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
}
// RunPVESigned is one signed Proxmox step (ring 1): the pve layer with the signed envelope, which the wrapper verifies
// itself, holding the /etc/pve write gate.
func (l *Leg) RunPVESigned(ctx context.Context, vmid int, p PVEStepParams, blob []byte, sig string) Report {
unlock := l.lockPass(true)
defer unlock()
l.sendUnsentLocked(ctx) // R-868
runID := l.now().UTC().Format("20060102T150405Z")
rid := p.ReleaseID
if rid == "" {
rid = "signed-" + runID
}
return l.runPVE(ctx, runID, vmid, "signed", l.Block(), dockerOpts{releaseID: rid, packages: p.Packages,
signed: map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}})
}
+12 -3
View File
@@ -145,21 +145,28 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
rep.OOMCheck = rawOrNil(wr.OOMCheck)
wantEngine := ""
rep.PVEManager = wr.PVEManager
wantEngine, wantPVE := "", ""
for _, u := range wr.Upgraded {
if u.Name == "docker-ce" {
wantEngine = EngineOf(u.Version)
}
if u.Name == "pve-manager" {
wantPVE = u.Version
}
}
verdict := func(h *Health) (bool, string) {
switch wr.Layer {
case LayerDocker:
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
case LayerHost:
case LayerHost, LayerPVE:
t := hub.TunnelUnknown
if l.Tunnel != nil {
t, _ = l.Tunnel.Status(ctx)
}
if wr.Layer == LayerPVE {
return PVEHealthVerdict(wr.HealthBefore, h, t, wantPVE, wr.PVEManager)
}
return HostHealthVerdict(wr.HealthBefore, h, t)
}
return HealthVerdict(wr.HealthBefore, h)
@@ -167,7 +174,7 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
ok, why := verdict(wr.HealthAfter)
if !ok && len(wr.Upgraded) > 0 && wr.VMID > 0 {
lane := "fast"
if wr.Layer == LayerDocker {
if wr.Layer == LayerDocker || wr.Layer == LayerPVE {
lane = "slow"
}
if hr, err := l.call(ctx, "kept-"+runID, map[string]any{"release_id": "kept", "layer": wr.Layer, "lane": lane,
@@ -187,6 +194,8 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
rep.RebootScanned = wr.RebootScanned
if wr.Layer == LayerDocker {
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
} else if wr.Layer == LayerPVE {
rep.Installed, rep.Pending = onlyPVE(wr.Installed), onlyPVEPending(wr.Pending)
} else {
planned := map[string]bool{}
for _, u := range wr.Upgraded {
+10
View File
@@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
"io"
"net/http"
"net/url"
@@ -105,6 +106,15 @@ func (c *Client) do(ctx context.Context, method, path string, body io.Reader, ou
// doBody is the single HTTP chokepoint: builds the request, sets auth, executes,
// maps non-2xx to APIError, and decodes the data envelope.
func (c *Client) doBody(ctx context.Context, method, path string, body io.Reader, contentType string, out any) error {
// R-812 option A: every non-GET call may write /etc/pve — it waits while a Proxmox package step restarts pmxcfs
// (pvegate). Pinned by TestPVEGate_ClientWriteWaitsGetDoesNot.
if method != http.MethodGet {
release, _, gerr := pvegate.Write(ctx)
if gerr != nil {
return fmt.Errorf("proxmox: %s %s held back by a Proxmox package step: %w", method, path, gerr)
}
defer release()
}
req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
if err != nil {
return fmt.Errorf("proxmox: building request: %w", err)
+33
View File
@@ -4,8 +4,10 @@ import (
"context"
"encoding/json"
"fmt"
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
"io"
"os/exec"
"path/filepath"
"strconv"
)
@@ -59,6 +61,15 @@ func (r *ExecRunner) Run(ctx context.Context, name string, args ...string) ([]by
// RunStdin is Run with the process stdin fed from stdin (nil = no stdin). The sudo-prefix/mode
// handling is identical to Run — kept here so both paths share one place.
func (r *ExecRunner) RunStdin(ctx context.Context, stdin io.Reader, name string, args ...string) ([]byte, []byte, error) {
// R-812 option A: a root CLI that writes /etc/pve waits while a Proxmox package step runs (pvegate).
// Pinned by TestPVEGate_ExecRunnerPctSetWaits / TestWritesEtcPVE.
if WritesEtcPVE(name, args) {
release, _, gerr := pvegate.Write(ctx)
if gerr != nil {
return nil, nil, fmt.Errorf("proxmox: %s held back by a Proxmox package step: %w", name, gerr)
}
defer release()
}
var cmd *exec.Cmd
if r.Mode == RunnerSudo {
sudo := r.SudoPath
@@ -77,6 +88,28 @@ func (r *ExecRunner) RunStdin(ctx context.Context, stdin io.Reader, name string,
return stdout.b, stderr.b, err
}
// WritesEtcPVE reports whether a root command writes /etc/pve: `pct` with a config-changing verb, `pvesm`, `pveum`,
// and the PBS storage wrapper's create / reconcile verbs. `pct exec|status|list|config` and every other command do not
// (the os-update wrapper itself must never wait on the gate its own step holds). Pinned by TestWritesEtcPVE.
func WritesEtcPVE(name string, args []string) bool {
base := filepath.Base(name)
switch base {
case "pvesm", "pveum":
return true
case "pct":
if len(args) == 0 {
return false
}
switch args[0] {
case "set", "create", "destroy", "restore", "unlock", "resize", "snapshot", "delsnapshot", "rollback", "move-volume", "start", "stop", "reboot", "shutdown":
return true
}
case "felhom-pbs-apply":
return len(args) > 0 && (args[0] == "create" || args[0] == "reconcile")
}
return false
}
// Privileged is the root-CLI backend.
type Privileged struct {
runner Runner
+92
View File
@@ -0,0 +1,92 @@
package proxmox
import (
"context"
"net/http"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
)
// R-812 option A: while a Proxmox package step holds the gate, a non-GET API call waits and a GET does not.
//
// COMPANION RED-PROOF (observed): delete the pvegate.Write block in doBody → this fails with "a PUT reached the API
// while the Proxmox step held the gate". Restored. (audits/day-2026-10-07/B/red-pvegate-chokepoints.txt)
func TestPVEGate_ClientWriteWaitsGetDoesNot(t *testing.T) {
d := &mockDoer{fn: func(*http.Request) (*http.Response, error) { return jsonResp(200, `{"data":null}`), nil }}
c := newTestClient(d)
end, err := pvegate.Step(context.Background())
if err != nil {
t.Fatal(err)
}
if err := c.get(context.Background(), "/nodes", nil); err != nil {
t.Fatalf("a GET must not wait on the gate: %v", err)
}
if d.calls != 1 {
t.Fatalf("the GET must reach the API, calls=%d", d.calls)
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
err = c.postForm(ctx, http.MethodPut, "/nodes/x/lxc/9201/config", nil, nil)
if d.calls != 1 {
end()
t.Fatal("a PUT reached the API while the Proxmox step held the gate")
}
if err == nil {
end()
t.Fatal("a PUT held back past its deadline must fail")
}
end()
if err := c.postForm(context.Background(), http.MethodPut, "/nodes/x/lxc/9201/config", nil, nil); err != nil || d.calls != 2 {
t.Fatalf("after the step the PUT must go through (err=%v calls=%d)", err, d.calls)
}
}
// A root `pct set` waits on the gate; `pct exec` does not.
//
// COMPANION RED-PROOF (observed): delete the WritesEtcPVE block in RunStdin → this fails with "pct set ran while the
// Proxmox step held the gate". Restored.
func TestPVEGate_ExecRunnerPctSetWaits(t *testing.T) {
r := &ExecRunner{Mode: RunnerDirect}
end, err := pvegate.Step(context.Background())
if err != nil {
t.Fatal(err)
}
defer end()
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
start := time.Now()
_, _, err = r.Run(ctx, "/nonexistent/pct", "set", "9201", "-mp8", "/x")
if err == nil || time.Since(start) < 90*time.Millisecond {
t.Fatalf("pct set ran while the Proxmox step held the gate (err=%v after %s)", err, time.Since(start))
}
start = time.Now()
_, _, _ = r.Run(context.Background(), "/nonexistent/pct", "exec", "9201", "--", "true")
if time.Since(start) > 80*time.Millisecond {
t.Fatal("pct exec must not wait on the gate")
}
}
func TestWritesEtcPVE(t *testing.T) {
for _, c := range []struct {
name string
args []string
want bool
}{
{"pct", []string{"set", "9201", "-mp8", "x"}, true},
{"/usr/sbin/pct", []string{"create", "9201"}, true},
{"pct", []string{"exec", "9201", "--", "true"}, false},
{"pct", []string{"status", "9201"}, false},
{"pvesm", []string{"add", "dir", "x"}, true},
{"pveum", []string{"acl", "modify"}, true},
{"/usr/local/sbin/felhom-pbs-apply", []string{"reconcile"}, true},
{"/usr/local/sbin/felhom-pbs-apply", []string{"read"}, false},
{"/usr/local/sbin/felhom-os-apply", []string{"--plan", "x"}, false},
{"pct", nil, false},
} {
if got := WritesEtcPVE(c.name, c.args); got != c.want {
t.Errorf("WritesEtcPVE(%s %v) = %v, want %v", c.name, c.args, got, c.want)
}
}
}
+104
View File
@@ -0,0 +1,104 @@
// Package pvegate keeps the agent's own writes to /etc/pve out of the way of a Proxmox package step (R-812 option A,
// `09` §3 decision 163, `11` §5.10).
//
// WHY. A `pve` step upgrades pve-cluster / pve-manager / qemu-server / pve-container; their postinst scripts restart
// pmxcfs (the FUSE filesystem behind /etc/pve) and the API daemons. A write that lands while pmxcfs restarts fails or,
// worse, half-lands (design-R-812 §3 A, "can go wrong"). Backups and restore-tests are already kept out by the
// host-wide heavy-op gate; this gate covers everything else the agent writes: every non-GET Proxmox API call
// (proxmox.Client.doBody) and every root CLI that writes /etc/pve (proxmox.ExecRunner — `pct set|create|…`, `pvesm`,
// `pveum`, `felhom-pbs-apply create|reconcile`).
//
// THE RULE. Write waits while a step runs (bounded by its own context). Step marks the step and then waits until every
// write already in flight has finished; it never waits forever (its context bounds it, and the caller gives up and
// does not run the step). One step at a time. Pinned by pvegate_test.go and, at the two chokepoints, by
// proxmox TestPVEGate_*.
package pvegate
import (
"context"
"errors"
"sync"
"time"
)
var (
mu sync.Mutex
inFlight int
stepping bool
stepDone chan struct{}
)
// ErrStepRunning is returned by Step when another step already holds the gate.
var ErrStepRunning = errors.New("pvegate: a Proxmox package step is already running")
// Write marks one /etc/pve write in flight, first waiting while a Proxmox package step runs. The returned release must
// be called when the write has finished. waited reports how long the write was held back.
func Write(ctx context.Context) (release func(), waited time.Duration, err error) {
start := time.Now()
for {
mu.Lock()
if !stepping {
inFlight++
mu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
mu.Lock()
inFlight--
mu.Unlock()
})
}, time.Since(start), nil
}
ch := stepDone
mu.Unlock()
select {
case <-ch:
case <-ctx.Done():
return nil, time.Since(start), ctx.Err()
}
}
}
// Step marks a Proxmox package step and waits until every /etc/pve write already in flight has finished. On error the
// gate is released again and the step must not run. end releases the gate and lets the held-back writes go.
func Step(ctx context.Context) (end func(), err error) {
mu.Lock()
if stepping {
mu.Unlock()
return nil, ErrStepRunning
}
stepping = true
done := make(chan struct{})
stepDone = done
mu.Unlock()
var once sync.Once
end = func() {
once.Do(func() {
mu.Lock()
stepping = false
close(done)
mu.Unlock()
})
}
for {
mu.Lock()
n := inFlight
mu.Unlock()
if n == 0 {
return end, nil
}
select {
case <-ctx.Done():
end()
return nil, ctx.Err()
case <-time.After(50 * time.Millisecond):
}
}
}
// Stepping reports whether a Proxmox package step holds the gate (for logs).
func Stepping() bool {
mu.Lock()
defer mu.Unlock()
return stepping
}
+104
View File
@@ -0,0 +1,104 @@
package pvegate
import (
"context"
"testing"
"time"
)
// A write that starts while a step runs waits until the step ends.
//
// COMPANION RED-PROOF (observed): make Write ignore `stepping` → this fails with "the write went through while the
// Proxmox step held the gate". Restored. (audits/day-2026-10-07/B/red-pvegate.txt)
func TestWrite_WaitsWhileAStepRuns(t *testing.T) {
end, err := Step(context.Background())
if err != nil {
t.Fatal(err)
}
got := make(chan time.Time, 1)
go func() {
rel, _, err := Write(context.Background())
if err == nil {
rel()
}
got <- time.Now()
}()
select {
case <-got:
end()
t.Fatal("the write went through while the Proxmox step held the gate")
case <-time.After(150 * time.Millisecond):
}
ended := time.Now()
end()
select {
case at := <-got:
if at.Before(ended) {
t.Fatal("the write finished before the step ended")
}
case <-time.After(2 * time.Second):
t.Fatal("the write never went through after the step ended")
}
}
// A step waits for a write already in flight before it starts.
func TestStep_WaitsForAWriteInFlight(t *testing.T) {
rel, _, err := Write(context.Background())
if err != nil {
t.Fatal(err)
}
started := make(chan struct{})
go func() {
end, err := Step(context.Background())
if err == nil {
close(started)
end()
}
}()
select {
case <-started:
rel()
t.Fatal("the step started while a write was in flight")
case <-time.After(150 * time.Millisecond):
}
rel()
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("the step never started after the write finished")
}
}
// A step that cannot drain the writes in time gives up and releases the gate (it never waits forever).
func TestStep_GivesUpAndReleases(t *testing.T) {
rel, _, _ := Write(context.Background())
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if _, err := Step(ctx); err == nil {
t.Fatal("the step must give up while a write is in flight past its deadline")
}
if Stepping() {
t.Fatal("a step that gave up must release the gate")
}
rel()
}
// A write held back past its own deadline returns the context's error.
func TestWrite_HonoursItsContext(t *testing.T) {
end, _ := Step(context.Background())
defer end()
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
if _, _, err := Write(ctx); err == nil {
t.Fatal("a write held back past its deadline must fail")
}
}
// One step at a time.
func TestStep_OneAtATime(t *testing.T) {
end, _ := Step(context.Background())
defer end()
if _, err := Step(context.Background()); err != ErrStepRunning {
t.Fatalf("a second step must be refused, got %v", err)
}
}
+5 -1
View File
@@ -52,6 +52,10 @@ const (
// (signed, operational key) like agent_update; the root wrapper re-verifies the same signature itself.
ClassOSDockerStep OpClass = "os_docker_step"
// A Proxmox package step on the host (R-812 option A, `11` §5.10) — ring 1. Destructive-class (signed, operational
// key) like os_docker_step; the root wrapper re-verifies the same signature itself.
ClassOSPVEStep OpClass = "os_pve_step"
// The config bundle (agent v0.143.0, R-840, `11` §5.4.2): the box's ROOT-OWNED files (sudoers, wrappers, units).
// Destructive-class (signed, operational key) like agent_update; the root wrapper re-verifies the signature itself.
ClassAgentConfigUpdate OpClass = "agent_config_update"
@@ -123,7 +127,7 @@ func Classify(class OpClass, prov Provenance) Disposition {
return Destructive
case ClassKeyRotation:
return Destructive
case ClassAgentUpdate, ClassOSDockerStep, ClassAgentConfigUpdate:
case ClassAgentUpdate, ClassOSDockerStep, ClassOSPVEStep, ClassAgentConfigUpdate:
// Never benign — no agent-internal provenance can make replacing the agent binary
// unsigned-safe (a compromised process must not be able to self-bless an update).
return Destructive