felhom-agent/internal/localapi/hostpoweroff.go

package localapi

import (
	"context"
	"errors"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"gitea.dooplex.hu/admin/felhom-agent/internal/poweroff"
)

// ── POST /host/poweroff (R-998, documentation/architecture/14-host-power-off.md) ──────────────────────
//
// The household's „A gép kikapcsolása": shut THIS guest down cleanly, then power the whole host off. The
// first local-API route whose effect reaches beyond the caller's own guest — a compromised controller can
// now switch the box off, and nobody can switch it on remotely. The mitigations are the limits below:
//
//   - refused in the first 10 minutes after boot (no power-off loop);
//   - at most ONE accepted request per boot (poweroff.State, persisted, keyed by the boot);
//   - refused while a backup, restore test, OS update step, controller update, drive format, NAS verify
//     or escrow ceremony runs, or a vzdump of the guest is in flight — each named in the 409;
//   - refused while ANY OTHER guest or VM runs on the host (security review 2026-10-13): one guest's
//     controller must not switch off another household's guest, a scratch guest or a test VM (demo-hp
//     carries the Tester 1 VM). Read from /proc (`lxc-start -n N`, `kvm -id N`), fail closed;
//   - the heavy-operation gate is TAKEN on acceptance and never given back (the box is going down), so no
//     backup, restore test or OS step can start between the checks and the power-off (security review).
//     Released only if the power-off command itself fails.
//
// Every check fails CLOSED: an unwired gate, an unreadable task list or /proc refuses.
// On acceptance the record is written FIRST, the hub is given a report carrying it (bounded wait), and
// only then: guest shutdown (time-limited), host power-off. Pinned by hostpoweroff_test.go.

// HostPowerer is the privileged half: both run as fixed sudo vectors (configs/felhom-agent.sudoers).
type HostPowerer interface {
	ShutdownGuest(ctx context.Context, vmid int, timeout time.Duration) error
	PowerOffHost(ctx context.Context) error
}

// HostPowerOffConfig wires the route. A nil config leaves it answering 503.
type HostPowerOffConfig struct {
	State     *poweroff.State
	Power     HostPowerer
	ReportNow func(ctx context.Context) bool // push a report carrying the record; true = the hub accepted it
	Uptime    func() (time.Duration, bool)   // time since host boot; false = unknown (refuse)
	Running   func() ([]int, error)          // vmids of every running guest and VM on the host; default /proc
	MinUptime time.Duration                  // default 10 min
	GuestStop time.Duration                  // pct shutdown --timeout; default 120 s
	ReportMax time.Duration                  // longest wait for the hub; default 60 s
	Now       func() time.Time
}

// Refusal codes — the controller maps each to a household sentence. Stable wire values.
const (
	PowerOffTooSoon      = "too_soon_after_boot"
	PowerOffAlreadyAsked = "already_requested_this_boot"
	PowerOffBackup       = "backup_running"
	PowerOffRestoreTest  = "restore_test_running"
	PowerOffOSUpdate     = "os_update_running"
	PowerOffCtrlUpdate   = "controller_update_running"
	PowerOffDrive        = "drive_operation_running"
	PowerOffEscrow       = "escrow_ceremony_running"
	PowerOffBusy         = "host_busy"
	PowerOffOtherGuests  = "other_guests_running"
	PowerOffUnknown      = "state_unknown"
)

// SetHostPowerOff enables POST /host/poweroff.
func (s *Server) SetHostPowerOff(c HostPowerOffConfig) {
	if c.MinUptime == 0 {
		c.MinUptime = 10 * time.Minute
	}
	if c.GuestStop == 0 {
		c.GuestStop = 120 * time.Second
	}
	if c.ReportMax == 0 {
		c.ReportMax = 60 * time.Second
	}
	if c.Now == nil {
		c.Now = time.Now
	}
	if c.Running == nil {
		c.Running = func() ([]int, error) { return runningGuestsFromProc("/proc") }
	}
	s.powerOff = &c
}

type hostPowerOffRequest struct {
	VMID int `json:"vmid"`
}

func (s *Server) handleHostPowerOff(w http.ResponseWriter, r *http.Request, vmid int) {
	c := s.powerOff
	if c == nil || c.State == nil || c.Power == nil {
		writeErr(w, http.StatusServiceUnavailable, "host power-off not configured on this host")
		return
	}
	if r.ContentLength != 0 {
		var req hostPowerOffRequest
		if !decodeBody(w, r, &req) {
			return
		}
		if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
			return
		}
	}
	// One decision at a time: two requests racing must not both pass the guards before either marks.
	s.powerOffMu.Lock()
	defer s.powerOffMu.Unlock()
	if code, detail := s.powerOffBlocker(r.Context(), vmid); code != "" {
		s.logger.Info("local-api: host power-off refused", "vmid", vmid, "reason", code, "detail", detail)
		writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": detail}, "host power-off refused: "+code)
		return
	}
	// Take the heavy-operation gate for good: nothing heavy may start between these checks and the power-off.
	release, busy, free := s.inFlight.TryAcquire("host-poweroff")
	if !free {
		writeStatus(w, http.StatusConflict, false, map[string]any{"reason": PowerOffBusy, "detail": busy}, "host power-off refused: "+PowerOffBusy)
		return
	}
	rec, err := c.State.Mark(c.Now(), vmid)
	if err != nil {
		release()
		code := PowerOffUnknown
		if errors.Is(err, poweroff.ErrAlreadyThisBoot) {
			code = PowerOffAlreadyAsked
		}
		s.logger.Warn("local-api: host power-off refused (record)", "vmid", vmid, "reason", code, "err", err)
		writeStatus(w, http.StatusConflict, false, map[string]any{"reason": code, "detail": err.Error()}, "host power-off refused: "+code)
		return
	}
	s.logger.Warn("local-api: HOST POWER-OFF accepted (household request)", "vmid", vmid,
		"requested_at", rec.RequestedAt.Format(time.RFC3339))
	base := s.baseCtx
	if base == nil {
		base = context.Background()
	}
	go s.runHostPowerOff(base, vmid, release)
	writeStatus(w, http.StatusAccepted, true, map[string]any{
		"vmid": vmid, "powering_off": true, "requested_at": rec.RequestedAt.Format(time.RFC3339),
	}, "")
}

// runHostPowerOff: hub first (bounded), then the guest (time-limited), then the host. A failed guest
// shutdown does not stop the power-off — the household asked for it, and Proxmox's own shutdown stops
// any guest still running (pve-guests) before the host goes down.
func (s *Server) runHostPowerOff(base context.Context, vmid int, release func()) {
	c := s.powerOff
	if c.ReportNow != nil {
		ctx, cancel := context.WithTimeout(base, c.ReportMax)
		ok := c.ReportNow(ctx)
		cancel()
		s.logger.Warn("host-poweroff: step 1/3 — hub told", "vmid", vmid, "hub_accepted", ok)
	}
	start := time.Now()
	ctx, cancel := context.WithTimeout(base, c.GuestStop+60*time.Second)
	err := c.Power.ShutdownGuest(ctx, vmid, c.GuestStop)
	cancel()
	if err != nil {
		s.logger.Error("host-poweroff: step 2/3 — guest shutdown failed; powering off anyway", "vmid", vmid,
			"took", time.Since(start).Round(time.Second), "err", err)
	} else {
		s.logger.Warn("host-poweroff: step 2/3 — guest stopped", "vmid", vmid, "took", time.Since(start).Round(time.Second))
	}
	s.logger.Warn("host-poweroff: step 3/3 — powering the host off")
	ctx, cancel = context.WithTimeout(base, 60*time.Second)
	defer cancel()
	if err := c.Power.PowerOffHost(ctx); err != nil {
		s.logger.Error("host-poweroff: host power-off command failed — heavy-operation gate released", "err", err)
		release()
	}
}

// powerOffBlocker returns the first reason to refuse ("" = none) and a detail for the log/operator.
func (s *Server) powerOffBlocker(ctx context.Context, vmid int) (string, string) {
	c := s.powerOff
	if c.State.Current() != nil {
		return PowerOffAlreadyAsked, "a power-off was already accepted in this boot"
	}
	if c.Uptime == nil {
		return PowerOffUnknown, "uptime not wired"
	}
	up, ok := c.Uptime()
	if !ok {
		return PowerOffUnknown, "cannot read the host uptime"
	}
	if up < c.MinUptime {
		return PowerOffTooSoon, "host up " + up.Round(time.Second).String() + ", minimum " + c.MinUptime.String()
	}
	if s.inFlight == nil || s.staleLock == nil {
		return PowerOffUnknown, "the heavy-operation gate or the Proxmox task reader is not wired"
	}
	running, err := c.Running()
	if err != nil {
		return PowerOffUnknown, "cannot list the running guests: " + err.Error()
	}
	var others []string
	for _, v := range running {
		if v != vmid {
			others = append(others, strconv.Itoa(v))
		}
	}
	if len(others) > 0 {
		return PowerOffOtherGuests, "other guests or VMs running: " + strings.Join(others, ", ")
	}
	if busy := s.inFlight.Busy(); busy != "" {
		switch {
		case strings.HasPrefix(busy, "backup:"):
			return PowerOffBackup, busy
		case busy == "restore-test":
			return PowerOffRestoreTest, busy
		case strings.HasPrefix(busy, "os-"):
			return PowerOffOSUpdate, busy
		default:
			return PowerOffBusy, busy
		}
	}
	s.jobsMu.Lock()
	for k, j := range s.jobs {
		if j != nil && backupInFlight(j.Phase) {
			s.jobsMu.Unlock()
			return PowerOffBackup, "backup job " + j.JobID + " on target " + k.target
		}
	}
	s.jobsMu.Unlock()
	s.swapMu.Lock()
	for v, on := range s.swapInFlight {
		if on {
			s.swapMu.Unlock()
			return PowerOffCtrlUpdate, "controller swap in flight for guest " + strconv.Itoa(v)
		}
	}
	s.swapMu.Unlock()
	if s.formatJobs != nil {
		if j := s.formatJobs.get(); j != nil && j.Phase == formatPhaseRunning {
			return PowerOffDrive, "drive format running"
		}
	}
	s.netVerifyMu.Lock()
	nv := s.netVerifyCur != nil && s.netVerifyCur.Phase == netVerifyPhaseRunning
	s.netVerifyMu.Unlock()
	if nv {
		return PowerOffDrive, "network storage verify running"
	}
	s.escrowMu.Lock()
	esc := s.escrowJob != nil && s.escrowJob.Phase == escrowPhaseRunning
	s.escrowMu.Unlock()
	if esc {
		return PowerOffEscrow, "escrow ceremony running"
	}
	vz, err := s.staleLock.BackupRunning(ctx, vmid)
	if err != nil {
		return PowerOffUnknown, "cannot list running Proxmox tasks: " + err.Error()
	}
	if vz {
		return PowerOffBackup, "vzdump running for guest " + strconv.Itoa(vmid)
	}
	return "", ""
}

// runningGuestsFromProc lists the vmids of running containers (`lxc-start … -n N`) and VMs (`kvm -id N`) from
// the process table — readable by the non-root agent, unlike /run/qemu-server (measured on demo-hp 2026-10-13).
// An unreadable /proc is an error (fail closed); an unreadable single process is skipped (it exited).
func runningGuestsFromProc(procDir string) ([]int, error) {
	ents, err := os.ReadDir(procDir)
	if err != nil {
		return nil, err
	}
	seen := map[int]bool{}
	var out []int
	for _, e := range ents {
		if _, err := strconv.Atoi(e.Name()); err != nil {
			continue
		}
		raw, err := os.ReadFile(filepath.Join(procDir, e.Name(), "cmdline"))
		if err != nil || len(raw) == 0 {
			continue
		}
		args := strings.Split(strings.TrimRight(string(raw), "\x00"), "\x00")
		flag := ""
		switch filepath.Base(args[0]) {
		case "lxc-start":
			flag = "-n"
		case "kvm", "qemu-system-x86_64":
			flag = "-id"
		default:
			continue
		}
		for i := 1; i+1 < len(args); i++ {
			if args[i] == flag {
				if v, err := strconv.Atoi(args[i+1]); err == nil && !seen[v] {
					seen[v] = true
					out = append(out, v)
				}
			}
		}
	}
	return out, nil
}