Host power-off: re-check jobs before each step; cancel if one is still running (R-998)
gates / gates (push) Successful in 29s
gates / gates (push) Successful in 29s
Security review of f38b03c: a controller update, drive/NAS job, escrow ceremony or Proxmox
vzdump could start between acceptance and the power-off (the heavy gate does not cover them).
runHostPowerOff now asks jobBlocker again before the guest shutdown and before the host
power-off, waits up to 15 min, and CANCELS (gate released, box stays on) if still busy.
Red-proofed: no wait, and a gate kept after a cancel, each convicted.
This commit is contained in:
@@ -50,6 +50,8 @@ type HostPowerOffConfig struct {
|
||||
MinUptime time.Duration // default 10 min
|
||||
GuestStop time.Duration // pct shutdown --timeout; default 120 s
|
||||
ReportMax time.Duration // longest wait for the hub; default 60 s
|
||||
JobWait time.Duration // longest wait for a job that started after acceptance; default 15 min
|
||||
JobPoll time.Duration // how often that wait looks again; default 10 s
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
@@ -82,6 +84,12 @@ func (s *Server) SetHostPowerOff(c HostPowerOffConfig) {
|
||||
if c.Now == nil {
|
||||
c.Now = time.Now
|
||||
}
|
||||
if c.JobWait == 0 {
|
||||
c.JobWait = 15 * time.Minute
|
||||
}
|
||||
if c.JobPoll == 0 {
|
||||
c.JobPoll = 10 * time.Second
|
||||
}
|
||||
if c.Running == nil {
|
||||
c.Running = func() ([]int, error) { return runningGuestsFromProc("/proc") }
|
||||
}
|
||||
@@ -155,6 +163,10 @@ func (s *Server) runHostPowerOff(base context.Context, vmid int, release func())
|
||||
cancel()
|
||||
s.logger.Warn("host-poweroff: step 1/3 — hub told", "vmid", vmid, "hub_accepted", ok)
|
||||
}
|
||||
if !s.waitJobsClear(base, vmid, "before the guest shutdown") {
|
||||
release()
|
||||
return
|
||||
}
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(base, c.GuestStop+60*time.Second)
|
||||
err := c.Power.ShutdownGuest(ctx, vmid, c.GuestStop)
|
||||
@@ -165,6 +177,10 @@ func (s *Server) runHostPowerOff(base context.Context, vmid int, release func())
|
||||
} else {
|
||||
s.logger.Warn("host-poweroff: step 2/3 — guest stopped", "vmid", vmid, "took", time.Since(start).Round(time.Second))
|
||||
}
|
||||
if !s.waitJobsClear(base, vmid, "before the host power-off") {
|
||||
release()
|
||||
return
|
||||
}
|
||||
s.logger.Warn("host-poweroff: step 3/3 — powering the host off")
|
||||
ctx, cancel = context.WithTimeout(base, 60*time.Second)
|
||||
defer cancel()
|
||||
@@ -218,6 +234,13 @@ func (s *Server) powerOffBlocker(ctx context.Context, vmid int) (string, string)
|
||||
return PowerOffBusy, busy
|
||||
}
|
||||
}
|
||||
return s.jobBlocker(ctx, vmid)
|
||||
}
|
||||
|
||||
// jobBlocker is the part of the checks that can change after acceptance — a backup job, a controller update, a drive
|
||||
// or NAS job, an escrow ceremony, a Proxmox vzdump. runHostPowerOff asks it again before each step (security review
|
||||
// 2026-10-13: the checks ran only once). The heavy-operation gate is not here: the power-off holds it.
|
||||
func (s *Server) jobBlocker(ctx context.Context, vmid int) (string, string) {
|
||||
s.jobsMu.Lock()
|
||||
for k, j := range s.jobs {
|
||||
if j != nil && backupInFlight(j.Phase) {
|
||||
@@ -300,3 +323,26 @@ func runningGuestsFromProc(procDir string) ([]int, error) {
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// waitJobsClear waits until jobBlocker finds nothing, at most JobWait. false = still busy (or unreadable) at the end:
|
||||
// the power-off is CANCELLED — the box stays on, the gate is released, the log says why. Better a box that did not
|
||||
// switch off than a backup, update or format cut in the middle.
|
||||
func (s *Server) waitJobsClear(base context.Context, vmid int, when string) bool {
|
||||
c := s.powerOff
|
||||
ctx, cancel := context.WithTimeout(base, c.JobWait)
|
||||
defer cancel()
|
||||
for {
|
||||
code, detail := s.jobBlocker(ctx, vmid)
|
||||
if code == "" {
|
||||
return true
|
||||
}
|
||||
s.logger.Warn("host-poweroff: waiting — a job started after the request", "when", when, "reason", code, "detail", detail)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.logger.Error("host-poweroff: CANCELLED — still busy after the wait; the box stays on", "when", when,
|
||||
"reason", code, "detail", detail, "waited", c.JobWait)
|
||||
return false
|
||||
case <-time.After(c.JobPoll):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user