Host power-off: refuse while another guest or VM runs; hold the heavy gate; fail closed (R-998)
gates / gates (push) Successful in 33s
gates / gates (push) Successful in 33s
Security review of 3403c87: (1) any guest's controller could switch off a shared host — demo-hp
runs a scratch guest and the Tester 1 VM. Now refused (other_guests_running) when /proc shows any
other lxc-start -n N or kvm -id N; an unreadable /proc refuses. (2) the checks ran once, so a
backup could start before the power-off: the heavy-operation gate is now taken on acceptance and
kept (released only if the power-off command fails). (3) an unwired gate or task reader refused
nothing; now refuses. Each red-proofed.
This commit is contained in:
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -20,8 +22,15 @@ import (
|
||||
// - refused in the first 10 minutes after boot (no power-off loop);
|
||||
// - at most ONE accepted request per boot (poweroff.State, persisted, keyed by the boot);
|
||||
// - refused while a backup, restore test, OS update step, controller update, drive format, NAS verify
|
||||
// or escrow ceremony runs, or a vzdump of the guest is in flight — each named in the 409.
|
||||
// or escrow ceremony runs, or a vzdump of the guest is in flight — each named in the 409;
|
||||
// - refused while ANY OTHER guest or VM runs on the host (security review 2026-10-13): one guest's
|
||||
// controller must not switch off another household's guest, a scratch guest or a test VM (demo-hp
|
||||
// carries the Tester 1 VM). Read from /proc (`lxc-start -n N`, `kvm -id N`), fail closed;
|
||||
// - the heavy-operation gate is TAKEN on acceptance and never given back (the box is going down), so no
|
||||
// backup, restore test or OS step can start between the checks and the power-off (security review).
|
||||
// Released only if the power-off command itself fails.
|
||||
//
|
||||
// Every check fails CLOSED: an unwired gate, an unreadable task list or /proc refuses.
|
||||
// On acceptance the record is written FIRST, the hub is given a report carrying it (bounded wait), and
|
||||
// only then: guest shutdown (time-limited), host power-off. Pinned by hostpoweroff_test.go.
|
||||
|
||||
@@ -37,6 +46,7 @@ type HostPowerOffConfig struct {
|
||||
Power HostPowerer
|
||||
ReportNow func(ctx context.Context) bool // push a report carrying the record; true = the hub accepted it
|
||||
Uptime func() (time.Duration, bool) // time since host boot; false = unknown (refuse)
|
||||
Running func() ([]int, error) // vmids of every running guest and VM on the host; default /proc
|
||||
MinUptime time.Duration // default 10 min
|
||||
GuestStop time.Duration // pct shutdown --timeout; default 120 s
|
||||
ReportMax time.Duration // longest wait for the hub; default 60 s
|
||||
@@ -54,6 +64,7 @@ const (
|
||||
PowerOffDrive = "drive_operation_running"
|
||||
PowerOffEscrow = "escrow_ceremony_running"
|
||||
PowerOffBusy = "host_busy"
|
||||
PowerOffOtherGuests = "other_guests_running"
|
||||
PowerOffUnknown = "state_unknown"
|
||||
)
|
||||
|
||||
@@ -71,6 +82,9 @@ func (s *Server) SetHostPowerOff(c HostPowerOffConfig) {
|
||||
if c.Now == nil {
|
||||
c.Now = time.Now
|
||||
}
|
||||
if c.Running == nil {
|
||||
c.Running = func() ([]int, error) { return runningGuestsFromProc("/proc") }
|
||||
}
|
||||
s.powerOff = &c
|
||||
}
|
||||
|
||||
@@ -101,8 +115,15 @@ func (s *Server) handleHostPowerOff(w http.ResponseWriter, r *http.Request, vmid
|
||||
writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": detail}, "host power-off refused: "+code)
|
||||
return
|
||||
}
|
||||
// Take the heavy-operation gate for good: nothing heavy may start between these checks and the power-off.
|
||||
release, busy, free := s.inFlight.TryAcquire("host-poweroff")
|
||||
if !free {
|
||||
writeStatus(w, http.StatusConflict, false, map[string]any{"reason": PowerOffBusy, "detail": busy}, "host power-off refused: "+PowerOffBusy)
|
||||
return
|
||||
}
|
||||
rec, err := c.State.Mark(c.Now(), vmid)
|
||||
if err != nil {
|
||||
release()
|
||||
code := PowerOffUnknown
|
||||
if errors.Is(err, poweroff.ErrAlreadyThisBoot) {
|
||||
code = PowerOffAlreadyAsked
|
||||
@@ -117,7 +138,7 @@ func (s *Server) handleHostPowerOff(w http.ResponseWriter, r *http.Request, vmid
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
go s.runHostPowerOff(base, vmid)
|
||||
go s.runHostPowerOff(base, vmid, release)
|
||||
writeStatus(w, http.StatusAccepted, true, map[string]any{
|
||||
"vmid": vmid, "powering_off": true, "requested_at": rec.RequestedAt.Format(time.RFC3339),
|
||||
}, "")
|
||||
@@ -126,7 +147,7 @@ func (s *Server) handleHostPowerOff(w http.ResponseWriter, r *http.Request, vmid
|
||||
// runHostPowerOff: hub first (bounded), then the guest (time-limited), then the host. A failed guest
|
||||
// shutdown does not stop the power-off — the household asked for it, and Proxmox's own shutdown stops
|
||||
// any guest still running (pve-guests) before the host goes down.
|
||||
func (s *Server) runHostPowerOff(base context.Context, vmid int) {
|
||||
func (s *Server) runHostPowerOff(base context.Context, vmid int, release func()) {
|
||||
c := s.powerOff
|
||||
if c.ReportNow != nil {
|
||||
ctx, cancel := context.WithTimeout(base, c.ReportMax)
|
||||
@@ -148,7 +169,8 @@ func (s *Server) runHostPowerOff(base context.Context, vmid int) {
|
||||
ctx, cancel = context.WithTimeout(base, 60*time.Second)
|
||||
defer cancel()
|
||||
if err := c.Power.PowerOffHost(ctx); err != nil {
|
||||
s.logger.Error("host-poweroff: host power-off command failed", "err", err)
|
||||
s.logger.Error("host-poweroff: host power-off command failed — heavy-operation gate released", "err", err)
|
||||
release()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,6 +190,22 @@ func (s *Server) powerOffBlocker(ctx context.Context, vmid int) (string, string)
|
||||
if up < c.MinUptime {
|
||||
return PowerOffTooSoon, "host up " + up.Round(time.Second).String() + ", minimum " + c.MinUptime.String()
|
||||
}
|
||||
if s.inFlight == nil || s.staleLock == nil {
|
||||
return PowerOffUnknown, "the heavy-operation gate or the Proxmox task reader is not wired"
|
||||
}
|
||||
running, err := c.Running()
|
||||
if err != nil {
|
||||
return PowerOffUnknown, "cannot list the running guests: " + err.Error()
|
||||
}
|
||||
var others []string
|
||||
for _, v := range running {
|
||||
if v != vmid {
|
||||
others = append(others, strconv.Itoa(v))
|
||||
}
|
||||
}
|
||||
if len(others) > 0 {
|
||||
return PowerOffOtherGuests, "other guests or VMs running: " + strings.Join(others, ", ")
|
||||
}
|
||||
if busy := s.inFlight.Busy(); busy != "" {
|
||||
switch {
|
||||
case strings.HasPrefix(busy, "backup:"):
|
||||
@@ -213,14 +251,52 @@ func (s *Server) powerOffBlocker(ctx context.Context, vmid int) (string, string)
|
||||
if esc {
|
||||
return PowerOffEscrow, "escrow ceremony running"
|
||||
}
|
||||
if s.staleLock != nil {
|
||||
running, err := s.staleLock.BackupRunning(ctx, vmid)
|
||||
if err != nil {
|
||||
return PowerOffUnknown, "cannot list running Proxmox tasks: " + err.Error()
|
||||
}
|
||||
if running {
|
||||
return PowerOffBackup, "vzdump running for guest " + strconv.Itoa(vmid)
|
||||
}
|
||||
vz, err := s.staleLock.BackupRunning(ctx, vmid)
|
||||
if err != nil {
|
||||
return PowerOffUnknown, "cannot list running Proxmox tasks: " + err.Error()
|
||||
}
|
||||
if vz {
|
||||
return PowerOffBackup, "vzdump running for guest " + strconv.Itoa(vmid)
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// runningGuestsFromProc lists the vmids of running containers (`lxc-start … -n N`) and VMs (`kvm -id N`) from
|
||||
// the process table — readable by the non-root agent, unlike /run/qemu-server (measured on demo-hp 2026-10-13).
|
||||
// An unreadable /proc is an error (fail closed); an unreadable single process is skipped (it exited).
|
||||
func runningGuestsFromProc(procDir string) ([]int, error) {
|
||||
ents, err := os.ReadDir(procDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[int]bool{}
|
||||
var out []int
|
||||
for _, e := range ents {
|
||||
if _, err := strconv.Atoi(e.Name()); err != nil {
|
||||
continue
|
||||
}
|
||||
raw, err := os.ReadFile(filepath.Join(procDir, e.Name(), "cmdline"))
|
||||
if err != nil || len(raw) == 0 {
|
||||
continue
|
||||
}
|
||||
args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")
|
||||
flag := ""
|
||||
switch filepath.Base(args[0]) {
|
||||
case "lxc-start":
|
||||
flag = "-n"
|
||||
case "kvm", "qemu-system-x86_64":
|
||||
flag = "-id"
|
||||
default:
|
||||
continue
|
||||
}
|
||||
for i := 1; i+1 < len(args); i++ {
|
||||
if args[i] == flag {
|
||||
if v, err := strconv.Atoi(args[i+1]); err == nil && !seen[v] {
|
||||
seen[v] = true
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user