Host power-off for the household (R-998): POST /host/poweroff
gates / gates (push) Successful in 34s
gates / gates (push) Successful in 34s
The guest's controller asks; the agent refuses in the first 10 min after boot, a second time in one boot (persisted, keyed by /proc/stat btime), and while a backup, restore test, OS step, controller update, drive format, NAS verify, escrow ceremony or vzdump runs (each a named 409 reason). On acceptance: the record first, a host report carrying deliberate_off accepted by the hub (bounded 60 s), then pct shutdown --timeout 120, then systemctl poweroff. Sudoers: two exact vectors (pct ^shutdown N --timeout N$, systemctl poweroff) in FELHOM_POWEROFF; nine injection cases. New TestEveryAliasIsGranted: a defined alias missing from the grant line passed the coverage check before. Every new test red-proofed.
This commit is contained in:
@@ -0,0 +1,226 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/poweroff"
|
||||
)
|
||||
|
||||
// ── POST /host/poweroff (R-998, documentation/architecture/14-host-power-off.md) ──────────────────────
|
||||
//
|
||||
// The household's „A gép kikapcsolása": shut THIS guest down cleanly, then power the whole host off. The
|
||||
// first local-API route whose effect reaches beyond the caller's own guest — a compromised controller can
|
||||
// now switch the box off, and nobody can switch it on remotely. The mitigations are the limits below:
|
||||
//
|
||||
// - refused in the first 10 minutes after boot (no power-off loop);
|
||||
// - at most ONE accepted request per boot (poweroff.State, persisted, keyed by the boot);
|
||||
// - refused while a backup, restore test, OS update step, controller update, drive format, NAS verify
|
||||
// or escrow ceremony runs, or a vzdump of the guest is in flight — each named in the 409.
|
||||
//
|
||||
// On acceptance the record is written FIRST, the hub is given a report carrying it (bounded wait), and
|
||||
// only then: guest shutdown (time-limited), host power-off. Pinned by hostpoweroff_test.go.
|
||||
|
||||
// HostPowerer is the privileged half: both run as fixed sudo vectors (configs/felhom-agent.sudoers).
|
||||
type HostPowerer interface {
|
||||
ShutdownGuest(ctx context.Context, vmid int, timeout time.Duration) error
|
||||
PowerOffHost(ctx context.Context) error
|
||||
}
|
||||
|
||||
// HostPowerOffConfig wires the route. A nil config leaves it answering 503.
|
||||
type HostPowerOffConfig struct {
|
||||
State *poweroff.State
|
||||
Power HostPowerer
|
||||
ReportNow func(ctx context.Context) bool // push a report carrying the record; true = the hub accepted it
|
||||
Uptime func() (time.Duration, bool) // time since host boot; false = unknown (refuse)
|
||||
MinUptime time.Duration // default 10 min
|
||||
GuestStop time.Duration // pct shutdown --timeout; default 120 s
|
||||
ReportMax time.Duration // longest wait for the hub; default 60 s
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
// Refusal codes — the controller maps each to a household sentence. Stable wire values.
|
||||
const (
|
||||
PowerOffTooSoon = "too_soon_after_boot"
|
||||
PowerOffAlreadyAsked = "already_requested_this_boot"
|
||||
PowerOffBackup = "backup_running"
|
||||
PowerOffRestoreTest = "restore_test_running"
|
||||
PowerOffOSUpdate = "os_update_running"
|
||||
PowerOffCtrlUpdate = "controller_update_running"
|
||||
PowerOffDrive = "drive_operation_running"
|
||||
PowerOffEscrow = "escrow_ceremony_running"
|
||||
PowerOffBusy = "host_busy"
|
||||
PowerOffUnknown = "state_unknown"
|
||||
)
|
||||
|
||||
// SetHostPowerOff enables POST /host/poweroff.
|
||||
func (s *Server) SetHostPowerOff(c HostPowerOffConfig) {
|
||||
if c.MinUptime == 0 {
|
||||
c.MinUptime = 10 * time.Minute
|
||||
}
|
||||
if c.GuestStop == 0 {
|
||||
c.GuestStop = 120 * time.Second
|
||||
}
|
||||
if c.ReportMax == 0 {
|
||||
c.ReportMax = 60 * time.Second
|
||||
}
|
||||
if c.Now == nil {
|
||||
c.Now = time.Now
|
||||
}
|
||||
s.powerOff = &c
|
||||
}
|
||||
|
||||
type hostPowerOffRequest struct {
|
||||
VMID int `json:"vmid"`
|
||||
}
|
||||
|
||||
func (s *Server) handleHostPowerOff(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
c := s.powerOff
|
||||
if c == nil || c.State == nil || c.Power == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "host power-off not configured on this host")
|
||||
return
|
||||
}
|
||||
if r.ContentLength != 0 {
|
||||
var req hostPowerOffRequest
|
||||
if !decodeBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
|
||||
return
|
||||
}
|
||||
}
|
||||
// One decision at a time: two requests racing must not both pass the guards before either marks.
|
||||
s.powerOffMu.Lock()
|
||||
defer s.powerOffMu.Unlock()
|
||||
if code, detail := s.powerOffBlocker(r.Context(), vmid); code != "" {
|
||||
s.logger.Info("local-api: host power-off refused", "vmid", vmid, "reason", code, "detail", detail)
|
||||
writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": detail}, "host power-off refused: "+code)
|
||||
return
|
||||
}
|
||||
rec, err := c.State.Mark(c.Now(), vmid)
|
||||
if err != nil {
|
||||
code := PowerOffUnknown
|
||||
if errors.Is(err, poweroff.ErrAlreadyThisBoot) {
|
||||
code = PowerOffAlreadyAsked
|
||||
}
|
||||
s.logger.Warn("local-api: host power-off refused (record)", "vmid", vmid, "reason", code, "err", err)
|
||||
writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": err.Error()}, "host power-off refused: "+code)
|
||||
return
|
||||
}
|
||||
s.logger.Warn("local-api: HOST POWER-OFF accepted (household request)", "vmid", vmid,
|
||||
"requested_at", rec.RequestedAt.Format(time.RFC3339))
|
||||
base := s.baseCtx
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
go s.runHostPowerOff(base, vmid)
|
||||
writeStatus(w, http.StatusAccepted, true, map[string]any{
|
||||
"vmid": vmid, "powering_off": true, "requested_at": rec.RequestedAt.Format(time.RFC3339),
|
||||
}, "")
|
||||
}
|
||||
|
||||
// runHostPowerOff: hub first (bounded), then the guest (time-limited), then the host. A failed guest
|
||||
// shutdown does not stop the power-off — the household asked for it, and Proxmox's own shutdown stops
|
||||
// any guest still running (pve-guests) before the host goes down.
|
||||
func (s *Server) runHostPowerOff(base context.Context, vmid int) {
|
||||
c := s.powerOff
|
||||
if c.ReportNow != nil {
|
||||
ctx, cancel := context.WithTimeout(base, c.ReportMax)
|
||||
ok := c.ReportNow(ctx)
|
||||
cancel()
|
||||
s.logger.Warn("host-poweroff: step 1/3 — hub told", "vmid", vmid, "hub_accepted", ok)
|
||||
}
|
||||
start := time.Now()
|
||||
ctx, cancel := context.WithTimeout(base, c.GuestStop+60*time.Second)
|
||||
err := c.Power.ShutdownGuest(ctx, vmid, c.GuestStop)
|
||||
cancel()
|
||||
if err != nil {
|
||||
s.logger.Error("host-poweroff: step 2/3 — guest shutdown failed; powering off anyway", "vmid", vmid,
|
||||
"took", time.Since(start).Round(time.Second), "err", err)
|
||||
} else {
|
||||
s.logger.Warn("host-poweroff: step 2/3 — guest stopped", "vmid", vmid, "took", time.Since(start).Round(time.Second))
|
||||
}
|
||||
s.logger.Warn("host-poweroff: step 3/3 — powering the host off")
|
||||
ctx, cancel = context.WithTimeout(base, 60*time.Second)
|
||||
defer cancel()
|
||||
if err := c.Power.PowerOffHost(ctx); err != nil {
|
||||
s.logger.Error("host-poweroff: host power-off command failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// powerOffBlocker returns the first reason to refuse ("" = none) and a detail for the log/operator.
|
||||
func (s *Server) powerOffBlocker(ctx context.Context, vmid int) (string, string) {
|
||||
c := s.powerOff
|
||||
if c.State.Current() != nil {
|
||||
return PowerOffAlreadyAsked, "a power-off was already accepted in this boot"
|
||||
}
|
||||
if c.Uptime == nil {
|
||||
return PowerOffUnknown, "uptime not wired"
|
||||
}
|
||||
up, ok := c.Uptime()
|
||||
if !ok {
|
||||
return PowerOffUnknown, "cannot read the host uptime"
|
||||
}
|
||||
if up < c.MinUptime {
|
||||
return PowerOffTooSoon, "host up " + up.Round(time.Second).String() + ", minimum " + c.MinUptime.String()
|
||||
}
|
||||
if busy := s.inFlight.Busy(); busy != "" {
|
||||
switch {
|
||||
case strings.HasPrefix(busy, "backup:"):
|
||||
return PowerOffBackup, busy
|
||||
case busy == "restore-test":
|
||||
return PowerOffRestoreTest, busy
|
||||
case strings.HasPrefix(busy, "os-"):
|
||||
return PowerOffOSUpdate, busy
|
||||
default:
|
||||
return PowerOffBusy, busy
|
||||
}
|
||||
}
|
||||
s.jobsMu.Lock()
|
||||
for k, j := range s.jobs {
|
||||
if j != nil && backupInFlight(j.Phase) {
|
||||
s.jobsMu.Unlock()
|
||||
return PowerOffBackup, "backup job " + j.JobID + " on target " + k.target
|
||||
}
|
||||
}
|
||||
s.jobsMu.Unlock()
|
||||
s.swapMu.Lock()
|
||||
for v, on := range s.swapInFlight {
|
||||
if on {
|
||||
s.swapMu.Unlock()
|
||||
return PowerOffCtrlUpdate, "controller swap in flight for guest " + strconv.Itoa(v)
|
||||
}
|
||||
}
|
||||
s.swapMu.Unlock()
|
||||
if s.formatJobs != nil {
|
||||
if j := s.formatJobs.get(); j != nil && j.Phase == formatPhaseRunning {
|
||||
return PowerOffDrive, "drive format running"
|
||||
}
|
||||
}
|
||||
s.netVerifyMu.Lock()
|
||||
nv := s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning
|
||||
s.netVerifyMu.Unlock()
|
||||
if nv {
|
||||
return PowerOffDrive, "network storage verify running"
|
||||
}
|
||||
s.escrowMu.Lock()
|
||||
esc := s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning
|
||||
s.escrowMu.Unlock()
|
||||
if esc {
|
||||
return PowerOffEscrow, "escrow ceremony running"
|
||||
}
|
||||
if s.staleLock != nil {
|
||||
running, err := s.staleLock.BackupRunning(ctx, vmid)
|
||||
if err != nil {
|
||||
return PowerOffUnknown, "cannot list running Proxmox tasks: " + err.Error()
|
||||
}
|
||||
if running {
|
||||
return PowerOffBackup, "vzdump running for guest " + strconv.Itoa(vmid)
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
Reference in New Issue
Block a user