R-812 option A: the Proxmox package lane (layer pve) + the /etc/pve write gate
The wrapper gains layer "pve" (slow lane): the host's Proxmox userspace packages only — origin "Proxmox Debian Repository", never a kernel / boot / firmware / microcode name (R14), no removal, no undo, a new package only from an allow-list; authority = a signed os_pve_step or the root-owned ring-0 mark. The night leg runs it in ring 0 after a healthy host step; ring 1 only by a signed job (PVEStepExecutor). While it runs, the agent's own /etc/pve writes (every non-GET API call, pct config verbs, pvesm, pveum, felhom-pbs-apply) wait on internal/pvegate. Health = the host rule + unchanged container ids + pveversion reads the installed pve-manager. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+131
-6
@@ -20,6 +20,7 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
|
||||
)
|
||||
|
||||
// WrapperPath is the pinned sudoers vector (configs/felhom-agent.sudoers FELHOM_OSAPPLY).
|
||||
@@ -40,8 +42,21 @@ const (
|
||||
LayerGuest = "guest"
|
||||
LayerHost = "host"
|
||||
LayerDocker = "docker" // the guest's Docker engine set — slow lane (`11` §5.8)
|
||||
// LayerPVE is the HOST's Proxmox userspace packages — slow lane (R-812 option A, `09` §3 decision 163, `11` §5.10):
|
||||
// ring 0 in the night leg after a healthy host step, ring 1 only inside a signed os_pve_step. Never a kernel.
|
||||
LayerPVE = "pve"
|
||||
)
|
||||
|
||||
// PVEOrigin is the origin apt prints for download.proxmox.com (the wrapper's PVE_ORIGIN); InstalledPVEOrigin is how
|
||||
// the wrapper's inventory names the same source.
|
||||
const (
|
||||
PVEOrigin = "Proxmox Debian Repository"
|
||||
InstalledPVEOrigin = "Proxmox"
|
||||
)
|
||||
|
||||
// hostSlowRE mirrors the wrapper's HOST_SLOW_RE: kernel, boot, firmware and microcode names never ride the pve lane.
|
||||
var hostSlowRE = regexp.MustCompile(`^(linux-(image|headers|kbuild|modules|base)|proxmox-kernel|proxmox-default-kernel|pve-kernel|pve-firmware|firmware-|grub|shim|systemd-boot|intel-microcode|amd64-microcode|efibootmgr)`)
|
||||
|
||||
// DockerNames are the six packages of the Docker engine set (the wrapper's DOCKER_NAMES).
|
||||
var DockerNames = map[string]bool{"containerd.io": true, "docker-buildx-plugin": true, "docker-ce": true,
|
||||
"docker-ce-cli": true, "docker-ce-rootless-extras": true, "docker-compose-plugin": true}
|
||||
@@ -109,6 +124,8 @@ type WrapperReport struct {
|
||||
// OOMCheck (R-528, `09` decision 157): the docker layer's memory-kill check, {result, oom_killed, oom_event,
|
||||
// exit_code, image, detail}. Carried to the hub UNCHANGED; the agent never reads it.
|
||||
OOMCheck json.RawMessage `json:"oom_check"`
|
||||
// PVEManager: the pve layer — pveversion's pve-manager version after the step ("unknown" = unreadable).
|
||||
PVEManager string `json:"pve_manager"`
|
||||
// R-868 (v0.144.0): the agent's ids, echoed from the plan, so a report kept on disk can be sent without the
|
||||
// agent process that started the pass. ReleaseID / VMID were always in the report.
|
||||
RunID string `json:"run_id"`
|
||||
@@ -149,6 +166,8 @@ type Report struct {
|
||||
// OOMCheck: docker layer — the wrapper's oom_check object, byte-for-byte (R-528; the hub decides approval on it).
|
||||
// Pinned by TestDocker_OOMCheckReachesTheHubUnchanged and TestR868_KeptCopyCarriesTheOOMCheck.
|
||||
OOMCheck json.RawMessage `json:"oom_check,omitempty"`
|
||||
// PVEManager: pve layer — pve-manager's version after the step (the hub's System page; R-812 option A).
|
||||
PVEManager string `json:"pve_manager,omitempty"`
|
||||
|
||||
unsent string // R-868: the wrapper's kept copy of this pass's report — deleted once the hub has it
|
||||
}
|
||||
@@ -386,6 +405,36 @@ func EngineOf(pkgVersion string) string {
|
||||
return v
|
||||
}
|
||||
|
||||
// PVEHealthVerdict is THE Proxmox-package-step health rule (R-812 option A; pinned by TestPVEHealthVerdict): the host
|
||||
// rule (every host service active, the guest running and healthy, the tunnel running), plus every container running at
|
||||
// the start still runs as the SAME container (a Proxmox step must not restart the household's apps), plus pveversion
|
||||
// now reports the pve-manager the step installed (wantPVE "" = pve-manager was not in the step).
|
||||
func PVEHealthVerdict(before, after *Health, tunnel, wantPVE, gotPVE string) (bool, string) {
|
||||
if ok, why := HostHealthVerdict(before, after, tunnel); !ok {
|
||||
return false, why
|
||||
}
|
||||
if before != nil && before.Guest != nil && after.Guest != nil {
|
||||
names := make([]string, 0, len(before.Guest.Containers))
|
||||
for n := range before.Guest.Containers {
|
||||
names = append(names, n)
|
||||
}
|
||||
sort.Strings(names)
|
||||
for _, n := range names {
|
||||
b := before.Guest.Containers[n]
|
||||
if b.State != "running" || b.ID == "" {
|
||||
continue
|
||||
}
|
||||
if a := after.Guest.Containers[n]; a.ID != b.ID {
|
||||
return false, n + " is a new container (id changed) — the Proxmox step restarted the household's app"
|
||||
}
|
||||
}
|
||||
}
|
||||
if wantPVE != "" && gotPVE != wantPVE {
|
||||
return false, "pveversion reads pve-manager " + gotPVE + ", not " + wantPVE
|
||||
}
|
||||
return true, ""
|
||||
}
|
||||
|
||||
// DockerHealthVerdict is THE Docker-step health rule (`11` §5.8; pinned by TestDockerHealthVerdict): the guest rule,
|
||||
// plus every container running at the start still runs as the SAME container (same id — a changed id means the
|
||||
// household's apps restarted, which `live-restore` exists to prevent), plus the engine now reports the version the step
|
||||
@@ -451,7 +500,7 @@ func (l *Leg) call(ctx context.Context, runID string, plan map[string]any) (Wrap
|
||||
|
||||
// Pass is one leg's reports; an empty Layer means the step did not run.
|
||||
type Pass struct {
|
||||
Guest, Host, Docker Report
|
||||
Guest, Host, Docker, PVE Report
|
||||
}
|
||||
|
||||
// Run is one pass: the guest layer, then (on an appliance, after a good guest step) the host layer, then (ring 0
|
||||
@@ -479,13 +528,44 @@ func (l *Leg) Run(ctx context.Context, vmid int, trigger string) Pass {
|
||||
if err := l.EnsureLiveRestore(ctx, g.RunID, vmid); err != nil {
|
||||
p.Docker = l.finish(ctx, lg, Report{RunID: g.RunID, Layer: LayerDocker, Trigger: trigger, Ring: 0, VMID: vmid,
|
||||
Mode: "apply", Outcome: "failed", HealthReason: "live-restore could not be turned on: " + err.Error()})
|
||||
return p
|
||||
} else {
|
||||
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
|
||||
}
|
||||
p.Docker = l.runLayer(ctx, g.RunID, LayerDocker, vmid, trigger, blk, dockerOpts{})
|
||||
}
|
||||
// R-812 option A: the Proxmox package step — ring 0, an appliance, after a HEALTHY host step (it is a host change).
|
||||
// A Docker step's outcome does not gate it (the Docker set lives in the guest). Pinned by TestPVE_*.
|
||||
switch {
|
||||
case blk.Ring != 0 || !blk.Enabled:
|
||||
lg.Info("osupdate: pve step skipped — ring 1 takes a Proxmox set only inside a signed operator job (`11` §5.10)", "ring", blk.Ring, "enabled", blk.Enabled)
|
||||
case !l.Appliance || h.Layer == "" || !okStep(h):
|
||||
lg.Info("osupdate: pve step skipped — no healthy host step this pass (an appliance only)", "appliance", l.Appliance, "host_outcome", h.Outcome)
|
||||
default:
|
||||
p.PVE = l.runPVE(ctx, g.RunID, vmid, trigger, blk, dockerOpts{})
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// pveDrainWait bounds how long a pve step waits for the agent's own /etc/pve writes in flight (pvegate).
|
||||
var pveDrainWait = 2 * time.Minute
|
||||
|
||||
// runPVE runs the pve layer while holding pvegate: the agent's own /etc/pve writes wait until it ends.
|
||||
func (l *Leg) runPVE(ctx context.Context, runID string, vmid int, trigger string, blk hub.WireOSUpdate, do dockerOpts) Report {
|
||||
lg := l.log().With("run", runID, "layer", LayerPVE, "vmid", vmid, "trigger", trigger)
|
||||
dctx, cancel := context.WithTimeout(ctx, pveDrainWait)
|
||||
end, err := pvegate.Step(dctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return l.finish(ctx, lg, Report{RunID: runID, Layer: LayerPVE, Trigger: trigger, Ring: blk.Ring, VMID: vmid, Mode: "apply",
|
||||
ReleaseID: do.releaseID, Outcome: "failed", HealthReason: "an agent write to /etc/pve did not finish in time (pvegate): " + err.Error()})
|
||||
}
|
||||
lg.Info("osupdate: pve step holds the /etc/pve write gate — the agent's own writes wait until it ends")
|
||||
defer func() {
|
||||
end()
|
||||
lg.Info("osupdate: pve step released the /etc/pve write gate")
|
||||
}()
|
||||
return l.runLayer(ctx, runID, LayerPVE, vmid, trigger, blk, do)
|
||||
}
|
||||
|
||||
// dockerOpts is a signed Docker step (DockerStepExecutor); the zero value is ring 0's unsigned "pending-docker".
|
||||
type dockerOpts struct {
|
||||
releaseID string
|
||||
@@ -571,7 +651,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
wire = blk.HostRelease
|
||||
}
|
||||
lane := "fast"
|
||||
if layer == LayerDocker {
|
||||
if layer == LayerDocker || layer == LayerPVE {
|
||||
lane = "slow"
|
||||
if do.signed != nil {
|
||||
rel = hub.WireOSRelease{ID: do.releaseID}
|
||||
@@ -604,6 +684,13 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
}
|
||||
case layer == LayerDocker:
|
||||
plan["select"] = "pending-docker" // ring 0: the wrapper checks the box's ROOT-OWNED ring-0 mark itself
|
||||
case layer == LayerPVE && do.signed != nil:
|
||||
plan["packages"], plan["signed"] = do.packages, do.signed
|
||||
for _, p := range do.packages {
|
||||
planned[p.Name] = true
|
||||
}
|
||||
case layer == LayerPVE:
|
||||
plan["select"] = "pending-pve" // ring 0: the same root-owned mark; the wrapper picks installed Proxmox userspace
|
||||
case !blk.Enabled:
|
||||
plan["mode"] = "inventory"
|
||||
lg.Info("osupdate: switched OFF for this box — reporting only")
|
||||
@@ -637,6 +724,7 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
|
||||
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
|
||||
rep.OOMCheck = rawOrNil(wr.OOMCheck)
|
||||
rep.PVEManager = wr.PVEManager
|
||||
if rep.Outcome == "" {
|
||||
switch {
|
||||
case rep.Mode == "inventory" && !blk.Enabled:
|
||||
@@ -654,21 +742,27 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
}
|
||||
// Health: compare with the start of the pass; give restarted services time (only after an install).
|
||||
cur := wr.HealthAfter
|
||||
wantEngine := ""
|
||||
wantEngine, wantPVE := "", ""
|
||||
for _, u := range wr.Upgraded {
|
||||
if u.Name == "docker-ce" {
|
||||
wantEngine = EngineOf(u.Version)
|
||||
}
|
||||
if u.Name == "pve-manager" {
|
||||
wantPVE = u.Version
|
||||
}
|
||||
}
|
||||
verdict := func(h *Health) (bool, string) {
|
||||
if layer == LayerDocker {
|
||||
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
|
||||
}
|
||||
if layer == LayerHost {
|
||||
if layer == LayerHost || layer == LayerPVE {
|
||||
t := hub.TunnelUnknown
|
||||
if l.Tunnel != nil {
|
||||
t, _ = l.Tunnel.Status(ctx)
|
||||
}
|
||||
if layer == LayerPVE {
|
||||
return PVEHealthVerdict(wr.HealthBefore, h, t, wantPVE, wr.PVEManager)
|
||||
}
|
||||
return HostHealthVerdict(wr.HealthBefore, h, t)
|
||||
}
|
||||
return HealthVerdict(wr.HealthBefore, h)
|
||||
@@ -708,6 +802,10 @@ func (l *Leg) runLayer(ctx context.Context, runID, layer string, vmid int, trigg
|
||||
// the docker report carries the engine set only (the guest report already carries the Debian packages)
|
||||
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
|
||||
rep.NotCovered = nil
|
||||
} else if layer == LayerPVE {
|
||||
// the pve report carries the Proxmox userspace set only — the hub's candidate is built from it
|
||||
rep.Installed, rep.Pending = onlyPVE(wr.Installed), onlyPVEPending(wr.Pending)
|
||||
rep.NotCovered = nil
|
||||
} else {
|
||||
rep.NotCovered = notCovered(wr.Pending, blk.Ring, planned)
|
||||
}
|
||||
@@ -732,6 +830,33 @@ func onlyDocker(in []Package) []Package {
|
||||
return out
|
||||
}
|
||||
|
||||
// onlyPVE keeps the installed Proxmox-origin packages the pve lane may touch (never a kernel / boot / firmware name).
|
||||
func onlyPVE(in []Package) []Package {
|
||||
var out []Package
|
||||
for _, p := range in {
|
||||
if (p.Origin == InstalledPVEOrigin || p.Origin == PVEOrigin) && !hostSlowRE.MatchString(p.Name) && !DockerNames[p.Name] {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func onlyPVEPending(in []Pending) []Pending {
|
||||
var out []Pending
|
||||
for _, p := range in {
|
||||
if p.From == "" || hostSlowRE.MatchString(p.Name) || DockerNames[p.Name] {
|
||||
continue
|
||||
}
|
||||
for _, o := range p.Origin {
|
||||
if o == PVEOrigin {
|
||||
out = append(out, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func onlyDockerPending(in []Pending) []Pending {
|
||||
var out []Pending
|
||||
for _, p := range in {
|
||||
|
||||
@@ -13,16 +13,18 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
|
||||
)
|
||||
|
||||
// fakeWrapper plays /usr/local/sbin/felhom-os-apply: it reads the plan the leg wrote and answers per layer and mode.
|
||||
type fakeWrapper struct {
|
||||
t *testing.T
|
||||
pending []Pending
|
||||
applyRep map[string]WrapperReport // per layer
|
||||
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
|
||||
plans []map[string]any
|
||||
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
|
||||
t *testing.T
|
||||
pending []Pending
|
||||
applyRep map[string]WrapperReport // per layer
|
||||
healthSeq map[string][]*Health // per layer: answers to successive "health" calls
|
||||
plans []map[string]any
|
||||
keep bool // R-868: like the real wrapper, keep an apply report beside the plan
|
||||
pveGateHeld bool
|
||||
}
|
||||
|
||||
func yes() *bool { b := true; return &b }
|
||||
@@ -50,9 +52,12 @@ func (f *fakeWrapper) Run(_ context.Context, name string, args ...string) ([]byt
|
||||
f.plans = append(f.plans, plan)
|
||||
layer := plan["layer"].(string)
|
||||
ok := guestOK()
|
||||
if layer == LayerHost {
|
||||
if layer == LayerHost || layer == LayerPVE {
|
||||
ok = hostOK()
|
||||
}
|
||||
if layer == LayerPVE && plan["mode"] == "apply" {
|
||||
f.pveGateHeld = pvegate.Stepping() // R-812: the /etc/pve write gate must be held while the pve step runs
|
||||
}
|
||||
var rep WrapperReport
|
||||
switch plan["mode"] {
|
||||
case "inventory":
|
||||
@@ -160,8 +165,8 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
|
||||
if g.Outcome != "applied" || !g.Healthy || ho.Outcome != "applied" || !ho.Healthy {
|
||||
t.Fatalf("guest %+v\nhost %+v", g, ho)
|
||||
}
|
||||
if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply" {
|
||||
t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore and the ring-0 docker step", calls(w))
|
||||
if calls(w) != "guest:apply,host:apply,guest:live-restore-on,docker:apply,pve:apply" {
|
||||
t.Fatalf("calls = %s, want one apply per layer, guest first, then live-restore, the ring-0 docker step and the pve step", calls(w))
|
||||
}
|
||||
for _, p := range w.plans[:2] {
|
||||
if p["select"] != "pending-fast" || p["snapshot"] != "" || len(p["packages"].([]any)) != 0 {
|
||||
@@ -171,7 +176,7 @@ func TestRing0_OneCallPerLayer(t *testing.T) {
|
||||
if len(g.NotCovered) != 1 || g.NotCovered[0] != "docker-ce" {
|
||||
t.Fatalf("not covered = %v", g.NotCovered)
|
||||
}
|
||||
if len(h.reports) != 3 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker {
|
||||
if len(h.reports) != 4 || h.reports[0].Layer != LayerGuest || h.reports[1].Layer != LayerHost || h.reports[2].Layer != LayerDocker || h.reports[3].Layer != LayerPVE {
|
||||
t.Fatalf("hub got %+v", h.reports)
|
||||
}
|
||||
}
|
||||
@@ -389,7 +394,7 @@ func TestHostReport_CarriesRebootScanned(t *testing.T) {
|
||||
}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
run2(l, "night")
|
||||
if len(h.reports) != 3 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
|
||||
if len(h.reports) != 4 || !h.reports[1].RebootScanned || !h.reports[1].RebootNeeded || h.reports[0].RebootScanned {
|
||||
t.Fatalf("hub got %+v", h.reports)
|
||||
}
|
||||
}
|
||||
@@ -436,7 +441,12 @@ func TestDocker_Ring0PlanAndReport(t *testing.T) {
|
||||
DockerEngine: "29.8.2", Authority: "ring0"}}}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
dp := w.plans[len(w.plans)-1]
|
||||
var dp map[string]any
|
||||
for _, x := range w.plans {
|
||||
if x["layer"] == "docker" {
|
||||
dp = x
|
||||
}
|
||||
}
|
||||
if dp["layer"] != "docker" || dp["lane"] != "slow" || dp["select"] != "pending-docker" {
|
||||
t.Fatalf("docker plan = %v", dp)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package osupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/hub"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/pvegate"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||
)
|
||||
|
||||
// ---- the Proxmox package step (R-812 option A, `09` §3 decision 163, `11` §5.10) ----
|
||||
|
||||
// Ring 0: after a healthy host step the leg runs the pve layer — slow lane, select pending-pve — while holding the
|
||||
// /etc/pve write gate; the report carries only Proxmox userspace packages and pve-manager's version.
|
||||
//
|
||||
// COMPANION RED-PROOF (observed): call runLayer instead of runPVE in Run → "the /etc/pve write gate was not held".
|
||||
func TestPVE_Ring0PlanGateAndReport(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {
|
||||
Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}},
|
||||
Installed: []Package{{Name: "pve-manager", Version: "9.2.21", Origin: "Proxmox"},
|
||||
{Name: "proxmox-kernel-helper", Version: "9.0.4", Origin: "Proxmox"}, {Name: "libc6", Version: "u4", Origin: "Debian"}},
|
||||
Pending: []Pending{{Name: "qemu-server", From: "9.0.1", To: "9.0.9", Origin: []string{PVEOrigin}},
|
||||
{Name: "proxmox-kernel-7.0", From: "7.0.2", To: "7.0.14", Origin: []string{PVEOrigin}},
|
||||
{Name: "libc6", From: "u3", To: "u4", Origin: []string{"Debian"}}},
|
||||
PVEManager: "9.2.21", Authority: "ring0"}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
var pp map[string]any
|
||||
for _, x := range w.plans {
|
||||
if x["layer"] == LayerPVE {
|
||||
pp = x
|
||||
}
|
||||
}
|
||||
if pp == nil || pp["lane"] != "slow" || pp["select"] != "pending-pve" {
|
||||
t.Fatalf("pve plan = %v (calls %s)", pp, calls(w))
|
||||
}
|
||||
if !w.pveGateHeld {
|
||||
t.Fatal("the /etc/pve write gate was not held while the pve step ran")
|
||||
}
|
||||
if pvegate.Stepping() {
|
||||
t.Fatal("the gate must be released after the step")
|
||||
}
|
||||
r := p.PVE
|
||||
if r.Outcome != "applied" || !r.Healthy || r.PVEManager != "9.2.21" {
|
||||
t.Fatalf("pve report = %+v", r)
|
||||
}
|
||||
if len(r.Installed) != 1 || r.Installed[0].Name != "pve-manager" || len(r.Pending) != 1 || r.Pending[0].Name != "qemu-server" {
|
||||
t.Fatalf("the pve report must carry Proxmox userspace only: installed=%v pending=%v", r.Installed, r.Pending)
|
||||
}
|
||||
if h.reports[len(h.reports)-1].Layer != LayerPVE {
|
||||
t.Fatalf("the hub must get the pve report: %+v", h.reports)
|
||||
}
|
||||
}
|
||||
|
||||
// Ring 1 never takes a Proxmox step in the night leg.
|
||||
func TestPVE_Ring1NightLegNeverSteps(t *testing.T) {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
if p := l.Run(context.Background(), 9201, "night"); p.PVE.Layer != "" || strings.Contains(calls(w), "pve") {
|
||||
t.Fatalf("ring 1 took a pve step: %s", calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// No healthy host step (a BYO box, or an unhealthy host step) → no pve step.
|
||||
func TestPVE_SkippedWithoutAHealthyHostStep(t *testing.T) {
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l.Appliance = false
|
||||
if p := l.Run(context.Background(), 9201, "night"); p.PVE.Layer != "" || strings.Contains(calls(w), "pve") {
|
||||
t.Fatalf("a BYO box took a pve step: %s", calls(w))
|
||||
}
|
||||
w2 := &fakeWrapper{t: t}
|
||||
l2, _ := newLeg(t, w2, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
l2.Tunnel = fakeTunnel{"stopped"} // the host step reads unhealthy
|
||||
w2.applyRep = map[string]WrapperReport{LayerHost: {Upgraded: []Package{{Name: "libc6", Version: "u4"}}}}
|
||||
if p := l2.Run(context.Background(), 9201, "night"); p.PVE.Layer != "" || strings.Contains(calls(w2), "pve") {
|
||||
t.Fatalf("a pve step ran after an unhealthy host step: %s", calls(w2))
|
||||
}
|
||||
}
|
||||
|
||||
// A write in flight that never finishes makes the pve step give up (failed), never run without the gate.
|
||||
func TestPVE_GivesUpWhenAWriteDoesNotFinish(t *testing.T) {
|
||||
old := pveDrainWait
|
||||
pveDrainWait = 50_000_000 // 50 ms
|
||||
defer func() { pveDrainWait = old }()
|
||||
rel, _, _ := pvegate.Write(context.Background())
|
||||
defer rel()
|
||||
w := &fakeWrapper{t: t}
|
||||
l, _ := newLeg(t, w, &hub.WireOSUpdate{Ring: 0, Enabled: true})
|
||||
p := l.Run(context.Background(), 9201, "night")
|
||||
if p.PVE.Outcome != "failed" || strings.Contains(calls(w), "pve") {
|
||||
t.Fatalf("the pve step must fail without a wrapper call: %+v calls=%s", p.PVE, calls(w))
|
||||
}
|
||||
}
|
||||
|
||||
// THE pve health rule. COMPANION RED-PROOF (observed): drop the container-id loop or the pve-manager check in
|
||||
// PVEHealthVerdict → the matching case below fails.
|
||||
func TestPVEHealthVerdict(t *testing.T) {
|
||||
before, after := hostOK(), hostOK()
|
||||
before.Guest.Containers["app"] = Container{State: "running", Health: "healthy", ID: "a1"}
|
||||
after.Guest.Containers["app"] = Container{State: "running", Health: "healthy", ID: "a1"}
|
||||
if ok, why := PVEHealthVerdict(before, after, hub.TunnelRunning, "9.2.21", "9.2.21"); !ok {
|
||||
t.Fatalf("healthy step read unhealthy: %s", why)
|
||||
}
|
||||
if ok, _ := PVEHealthVerdict(before, after, hub.TunnelRunning, "9.2.21", "9.2.2"); ok {
|
||||
t.Fatal("pveversion still on the old pve-manager must fail")
|
||||
}
|
||||
after.Guest.Containers["app"] = Container{State: "running", Health: "healthy", ID: "b2"}
|
||||
if ok, why := PVEHealthVerdict(before, after, hub.TunnelRunning, "", "9.2.2"); ok || !strings.Contains(why, "id changed") {
|
||||
t.Fatalf("an app restarted by the Proxmox step must fail, got ok=%v %q", ok, why)
|
||||
}
|
||||
}
|
||||
|
||||
// The signed executor hands the RAW envelope and the exact list to the wrapper's pve layer.
|
||||
func TestPVEStepExecutor_PassesTheSignedEnvelope(t *testing.T) {
|
||||
w := &fakeWrapper{t: t, applyRep: map[string]WrapperReport{LayerPVE: {
|
||||
Upgraded: []Package{{Name: "pve-manager", Version: "9.2.21"}}, PVEManager: "9.2.21", Authority: "signed"}}}
|
||||
l, h := newLeg(t, w, &hub.WireOSUpdate{Ring: 1, Enabled: true})
|
||||
e := PVEStepExecutor{Leg: l, Guest: func(context.Context) (int, error) { return 9201, nil }}
|
||||
params, _ := json.Marshal(PVEStepParams{ReleaseID: "os-pve-1", Packages: []Package{{Name: "pve-manager", Version: "9.2.21", Origin: PVEOrigin}}})
|
||||
ctx := signedjobs.WithSignedOp(context.Background(), &reconcile.SignedOp{Blob: []byte(`{"op":"os_pve_step"}`), Sig: []byte("SIG")})
|
||||
if err := e.Execute(ctx, OpPVEStep, params); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pp := w.plans[len(w.plans)-1]
|
||||
sg, _ := pp["signed"].(map[string]any)
|
||||
if pp["layer"] != LayerPVE || pp["lane"] != "slow" || pp["release_id"] != "os-pve-1" || sg == nil ||
|
||||
sg["blob_b64"] != base64.StdEncoding.EncodeToString([]byte(`{"op":"os_pve_step"}`)) || sg["sig"] != "SIG" {
|
||||
t.Fatalf("pve plan = %v", pp)
|
||||
}
|
||||
if !w.pveGateHeld || calls(w) != "pve:apply" || len(h.reports) != 1 || h.reports[0].Trigger != "signed" {
|
||||
t.Fatalf("gate=%v calls=%s reports=%+v", w.pveGateHeld, calls(w), h.reports)
|
||||
}
|
||||
if err := e.Execute(context.Background(), OpPVEStep, params); err == nil {
|
||||
t.Fatal("no envelope must refuse")
|
||||
}
|
||||
if err := e.Execute(context.Background(), OpDockerStep, params); err != signedjobs.ErrNoExecutor {
|
||||
t.Fatalf("another op must pass through the chain: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// os_pve_step is never benign.
|
||||
func TestPVEStep_IsDestructiveClass(t *testing.T) {
|
||||
if reconcile.Classify(reconcile.ClassOSPVEStep, reconcile.Provenance{}) != reconcile.Destructive {
|
||||
t.Fatal("os_pve_step must be destructive-class (signed, operational key)")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
package osupdate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/signedjobs"
|
||||
)
|
||||
|
||||
// OpPVEStep is the signed op class of a Proxmox package step (R-812 option A, `11` §5.10): a ring-1 box takes an
|
||||
// approved Proxmox set only through it. No undo in this release. CC may sign it until the first paying customer.
|
||||
const OpPVEStep = "os_pve_step"
|
||||
|
||||
// PVEStepParams are the signed params. The wrapper compares Packages with the plan byte-for-byte.
|
||||
type PVEStepParams struct {
|
||||
ReleaseID string `json:"release_id"`
|
||||
Packages []Package `json:"packages"`
|
||||
VMID int `json:"vmid,omitempty"`
|
||||
}
|
||||
|
||||
// PVEStepExecutor runs a verified os_pve_step (signedjobs.Executor) under the host-wide heavy-op gate (Gate) and the
|
||||
// /etc/pve write gate (inside runPVE).
|
||||
type PVEStepExecutor struct {
|
||||
Leg *Leg
|
||||
Guest func(ctx context.Context) (int, error)
|
||||
Gate func(ctx context.Context) (release func(), err error)
|
||||
}
|
||||
|
||||
// Execute implements signedjobs.Executor.
|
||||
func (e PVEStepExecutor) Execute(ctx context.Context, op string, params json.RawMessage) error {
|
||||
if op != OpPVEStep {
|
||||
return signedjobs.ErrNoExecutor
|
||||
}
|
||||
so, ok := signedjobs.SignedOpFrom(ctx)
|
||||
if !ok {
|
||||
return fmt.Errorf("os_pve_step: no signed envelope in the context — the wrapper could not verify it")
|
||||
}
|
||||
var p PVEStepParams
|
||||
if err := json.Unmarshal(params, &p); err != nil || len(p.Packages) == 0 {
|
||||
return fmt.Errorf("os_pve_step: params must name the Proxmox set: %v", err)
|
||||
}
|
||||
vmid := p.VMID
|
||||
if vmid == 0 {
|
||||
if e.Guest == nil {
|
||||
return fmt.Errorf("os_pve_step: no vmid and no guest finder")
|
||||
}
|
||||
v, err := e.Guest(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("os_pve_step: find the customer guest: %w", err)
|
||||
}
|
||||
vmid = v
|
||||
}
|
||||
if e.Gate != nil {
|
||||
release, err := e.Gate(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("os_pve_step: heavy-op gate busy (a backup or restore-test runs): %w", err)
|
||||
}
|
||||
defer release()
|
||||
}
|
||||
rep := e.Leg.RunPVESigned(ctx, vmid, p, so.Blob, string(so.Sig))
|
||||
switch rep.Outcome {
|
||||
case "applied", "nothing":
|
||||
if rep.Healthy {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("os_pve_step: %s (%s) %s", rep.Outcome, rep.HealthReason, string(rep.Refused))
|
||||
}
|
||||
|
||||
// RunPVESigned is one signed Proxmox step (ring 1): the pve layer with the signed envelope, which the wrapper verifies
|
||||
// itself, holding the /etc/pve write gate.
|
||||
func (l *Leg) RunPVESigned(ctx context.Context, vmid int, p PVEStepParams, blob []byte, sig string) Report {
|
||||
unlock := l.lockPass(true)
|
||||
defer unlock()
|
||||
l.sendUnsentLocked(ctx) // R-868
|
||||
runID := l.now().UTC().Format("20060102T150405Z")
|
||||
rid := p.ReleaseID
|
||||
if rid == "" {
|
||||
rid = "signed-" + runID
|
||||
}
|
||||
return l.runPVE(ctx, runID, vmid, "signed", l.Block(), dockerOpts{releaseID: rid, packages: p.Packages,
|
||||
signed: map[string]string{"blob_b64": base64.StdEncoding.EncodeToString(blob), "sig": sig}})
|
||||
}
|
||||
@@ -145,21 +145,28 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
|
||||
rep.Upgraded, rep.PassSeconds = wr.Upgraded, wr.PassSeconds
|
||||
rep.DockerEngine, rep.Authority, rep.Undo = wr.DockerEngine, wr.Authority, wr.Undo
|
||||
rep.OOMCheck = rawOrNil(wr.OOMCheck)
|
||||
wantEngine := ""
|
||||
rep.PVEManager = wr.PVEManager
|
||||
wantEngine, wantPVE := "", ""
|
||||
for _, u := range wr.Upgraded {
|
||||
if u.Name == "docker-ce" {
|
||||
wantEngine = EngineOf(u.Version)
|
||||
}
|
||||
if u.Name == "pve-manager" {
|
||||
wantPVE = u.Version
|
||||
}
|
||||
}
|
||||
verdict := func(h *Health) (bool, string) {
|
||||
switch wr.Layer {
|
||||
case LayerDocker:
|
||||
return DockerHealthVerdict(wr.HealthBefore, h, wantEngine, wr.DockerEngine)
|
||||
case LayerHost:
|
||||
case LayerHost, LayerPVE:
|
||||
t := hub.TunnelUnknown
|
||||
if l.Tunnel != nil {
|
||||
t, _ = l.Tunnel.Status(ctx)
|
||||
}
|
||||
if wr.Layer == LayerPVE {
|
||||
return PVEHealthVerdict(wr.HealthBefore, h, t, wantPVE, wr.PVEManager)
|
||||
}
|
||||
return HostHealthVerdict(wr.HealthBefore, h, t)
|
||||
}
|
||||
return HealthVerdict(wr.HealthBefore, h)
|
||||
@@ -167,7 +174,7 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
|
||||
ok, why := verdict(wr.HealthAfter)
|
||||
if !ok && len(wr.Upgraded) > 0 && wr.VMID > 0 {
|
||||
lane := "fast"
|
||||
if wr.Layer == LayerDocker {
|
||||
if wr.Layer == LayerDocker || wr.Layer == LayerPVE {
|
||||
lane = "slow"
|
||||
}
|
||||
if hr, err := l.call(ctx, "kept-"+runID, map[string]any{"release_id": "kept", "layer": wr.Layer, "lane": lane,
|
||||
@@ -187,6 +194,8 @@ func (l *Leg) reportFromKept(ctx context.Context, wr WrapperReport, path string)
|
||||
rep.RebootScanned = wr.RebootScanned
|
||||
if wr.Layer == LayerDocker {
|
||||
rep.Installed, rep.Pending = onlyDocker(wr.Installed), onlyDockerPending(wr.Pending)
|
||||
} else if wr.Layer == LayerPVE {
|
||||
rep.Installed, rep.Pending = onlyPVE(wr.Installed), onlyPVEPending(wr.Pending)
|
||||
} else {
|
||||
planned := map[string]bool{}
|
||||
for _, u := range wr.Upgraded {
|
||||
|
||||
Reference in New Issue
Block a user