v0.90.0 — guest RAM resize (R-24) + fast-tick-until-convergence (R-28)
MinAgent coupling: felhom-controller v0.143.0 gates its guest-memory-resize UI on this agent (FeatureGuestMemoryResize, MinAgent 0.90.0). R-24 guest RAM resize (internal/localapi/guestmemory.go): self-scoped GET/POST /guest/memory. Agent enforces every bound FRESH per request (min 2048, max host_total-2048, shrink floor max(2048, usage+512)); applies via PVE SetConfig — live cgroup apply, no reboot (Phase-0 proven on the nested demo box). Verify-after-apply re-reads maxmem before claiming success. New narrow MemoryOps seam (GuestAPI untouched); Options.Memory nil -> 503. Memory only. R-28 fast-tick (internal/fasttick): while any desired-state item is unapplied - including the pre-tunnel window a hub poke can't reach - pulse the shared out-of-band trigger every 30s, self-disarm on convergence. Four cached sources (desired-gen==0, reconcile Planned-Pending>0, pbsdr waiting_secret only, wgtunnel desired-not-operational); LOUD pbsdr states + pending_signature excluded. Seams: reconcile.Engine.LastResult() + wgtunnel.Manager.TunnelConvergence() (cached, no per-tick exec). Guests-0/0: hypothesis REFUTED live (9201 IS a pool member; 0/0 was the pre-provision window; PoolAddVMID re-assert already covers restore-over-existing). No code change; the fast-tick mitigates the window. Tests + red-proofs (i floor guard, ii max guard, iii always-pulse) all restored green.
This commit is contained in:
@@ -0,0 +1,259 @@
|
||||
package localapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
)
|
||||
|
||||
// Guest RAM resize (v0.90.0, R-24 controller-direct). The customer sees the guest's current memory
|
||||
// on the controller's system-settings page and resizes it; the controller calls these self-scoped
|
||||
// endpoints; the AGENT enforces every bound FRESH per request (the UI's numbers are decoration) and
|
||||
// applies via the PVE API's SetConfig — a live cgroup apply, no reboot (Phase-0 proved it: maxmem
|
||||
// moves with the guest running, /proc/meminfo ripples via lxcfs). Memory only; cores stay observation.
|
||||
//
|
||||
// Ruled bounds (Viktor, 2026-07-17):
|
||||
// - min = minGuestMemoryMB (2048)
|
||||
// - max = host_total − hostReserveMB (2048 reserved for the host)
|
||||
// - shrink floor = max(minGuestMemoryMB, current_usage + shrinkUsageMarginMB) — a customer wanting
|
||||
// less must stop applications first (the refusal says so, in the controller's Hungarian).
|
||||
//
|
||||
// The floor gates SHRINK only; a grow is bounded by min/max alone.
|
||||
|
||||
const (
|
||||
// minGuestMemoryMB is the ruled floor for any guest allocation (a Felhom box needs headroom).
|
||||
minGuestMemoryMB int64 = 2048
|
||||
// hostReserveMB is held back from the host total so a resize can never starve the hypervisor.
|
||||
hostReserveMB int64 = 2048
|
||||
// shrinkUsageMarginMB is the safety gap kept above live usage on a SHRINK (the "never below
|
||||
// current usage" ruling, with headroom). One named constant — trivially re-ruled.
|
||||
shrinkUsageMarginMB int64 = 512
|
||||
// mib is one Proxmox "memory" unit (MiB) in bytes. PVE config `memory` is MiB; status/node
|
||||
// memory fields are bytes — convert at this boundary (the §8 units trap).
|
||||
mib int64 = 1 << 20
|
||||
)
|
||||
|
||||
// MemoryOps is the guest-RAM-resize Proxmox surface (v0.90.0). Satisfied by *proxmox.Client. Kept
|
||||
// separate from GuestAPI so adding it breaks no existing fake. Every method is invoked ONLY with the
|
||||
// token-resolved VMID (never a caller-supplied id).
|
||||
type MemoryOps interface {
|
||||
GuestStatus(ctx context.Context, vmid int) (proxmox.Guest, error)
|
||||
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
||||
NodeStatus(ctx context.Context) (proxmox.NodeStatus, error)
|
||||
SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error)
|
||||
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
|
||||
}
|
||||
|
||||
// MemoryInfo is GET /guest/memory — every field in MB, computed agent-side (Scenario C). min/max/floor
|
||||
// are the CURRENT enforced bounds, so a stale UI re-renders honestly on every read/refusal.
|
||||
type MemoryInfo struct {
|
||||
VMID int `json:"vmid"`
|
||||
AllocatedMB int64 `json:"allocated_mb"`
|
||||
UsageMB int64 `json:"usage_mb"`
|
||||
HostTotalMB int64 `json:"host_total_mb"`
|
||||
MinMB int64 `json:"min_mb"`
|
||||
MaxMB int64 `json:"max_mb"`
|
||||
FloorMB int64 `json:"floor_mb"`
|
||||
Running bool `json:"running"`
|
||||
}
|
||||
|
||||
// memoryResizeRequest is the POST /guest/memory body.
|
||||
type memoryResizeRequest struct {
|
||||
VMID int `json:"vmid,omitempty"` // optional; if set must equal the token's guest (self-scope)
|
||||
MemoryMB int64 `json:"memory_mb"`
|
||||
}
|
||||
|
||||
// memoryRefusal is the data field of a 412 refusal: the machine code the controller maps to Hungarian,
|
||||
// plus the fresh bounds so the UI re-renders without a second round-trip.
|
||||
type memoryRefusal struct {
|
||||
Code string `json:"code"` // below_min | above_max | below_usage_floor
|
||||
AllocatedMB int64 `json:"allocated_mb"`
|
||||
UsageMB int64 `json:"usage_mb"`
|
||||
HostTotalMB int64 `json:"host_total_mb"`
|
||||
MinMB int64 `json:"min_mb"`
|
||||
MaxMB int64 `json:"max_mb"`
|
||||
FloorMB int64 `json:"floor_mb"`
|
||||
}
|
||||
|
||||
// memoryResizeResult is the success data field.
|
||||
type memoryResizeResult struct {
|
||||
VMID int `json:"vmid"`
|
||||
OldMB int64 `json:"old_mb"`
|
||||
NewMB int64 `json:"new_mb"`
|
||||
Unchanged bool `json:"unchanged"`
|
||||
}
|
||||
|
||||
// memoryBounds is one fresh snapshot of everything the validation needs.
|
||||
type memoryBounds struct {
|
||||
allocatedMB int64
|
||||
usageMB int64
|
||||
hostTotalMB int64
|
||||
minMB int64
|
||||
maxMB int64
|
||||
floorMB int64
|
||||
running bool
|
||||
}
|
||||
|
||||
func (b memoryBounds) refusal(code string) memoryRefusal {
|
||||
return memoryRefusal{
|
||||
Code: code, AllocatedMB: b.allocatedMB, UsageMB: b.usageMB, HostTotalMB: b.hostTotalMB,
|
||||
MinMB: b.minMB, MaxMB: b.maxMB, FloorMB: b.floorMB,
|
||||
}
|
||||
}
|
||||
|
||||
func (b memoryBounds) info(vmid int) MemoryInfo {
|
||||
return MemoryInfo{
|
||||
VMID: vmid, AllocatedMB: b.allocatedMB, UsageMB: b.usageMB, HostTotalMB: b.hostTotalMB,
|
||||
MinMB: b.minMB, MaxMB: b.maxMB, FloorMB: b.floorMB, Running: b.running,
|
||||
}
|
||||
}
|
||||
|
||||
// bytesToMBUp converts bytes → MB rounding UP (usage must never be under-reported for the floor).
|
||||
func bytesToMBUp(b int64) int64 {
|
||||
if b <= 0 {
|
||||
return 0
|
||||
}
|
||||
return (b + mib - 1) / mib
|
||||
}
|
||||
|
||||
// readMemoryBounds computes the enforced bounds from a FRESH read of guest config/status + host total.
|
||||
// Called by BOTH the GET and the POST so a stale UI can never smuggle an old max/floor.
|
||||
func (s *Server) readMemoryBounds(ctx context.Context, vmid int) (memoryBounds, error) {
|
||||
cfg, err := s.mem.GuestConfig(ctx, vmid)
|
||||
if err != nil {
|
||||
return memoryBounds{}, fmt.Errorf("guest config: %w", err)
|
||||
}
|
||||
st, err := s.mem.GuestStatus(ctx, vmid)
|
||||
if err != nil {
|
||||
return memoryBounds{}, fmt.Errorf("guest status: %w", err)
|
||||
}
|
||||
node, err := s.mem.NodeStatus(ctx)
|
||||
if err != nil {
|
||||
return memoryBounds{}, fmt.Errorf("node status: %w", err)
|
||||
}
|
||||
b := memoryBounds{
|
||||
allocatedMB: cfg.Memory, // PVE config memory is already MB
|
||||
usageMB: bytesToMBUp(st.Mem), // bytes → MB, rounded up
|
||||
hostTotalMB: node.Memory.Total / mib, // bytes → MB, floor (conservative for max)
|
||||
minMB: minGuestMemoryMB,
|
||||
running: st.Status == "running",
|
||||
}
|
||||
b.maxMB = b.hostTotalMB - hostReserveMB
|
||||
b.floorMB = b.usageMB + shrinkUsageMarginMB
|
||||
if b.floorMB < minGuestMemoryMB {
|
||||
b.floorMB = minGuestMemoryMB
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// handleGuestMemory serves GET /guest/memory — the current allocation, live usage, and the enforced
|
||||
// bounds, all agent-computed (Scenario C).
|
||||
func (s *Server) handleGuestMemory(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.mem == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "guest memory resize not configured on this host")
|
||||
return
|
||||
}
|
||||
b, err := s.readMemoryBounds(r.Context(), vmid)
|
||||
if err != nil {
|
||||
s.logger.Error("local-api: guest-memory read", "vmid", vmid, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "could not read guest memory: "+err.Error())
|
||||
return
|
||||
}
|
||||
writeOK(w, b.info(vmid))
|
||||
}
|
||||
|
||||
// handleGuestMemoryResize serves POST /guest/memory — validate FRESH against the ruled bounds (the UI's
|
||||
// numbers are decoration), then apply via SetConfig (live cgroup apply) and verify the new maxmem before
|
||||
// claiming success. Single-flight (one customer per host). SetConfig is NEVER called on a refusal path.
|
||||
func (s *Server) handleGuestMemoryResize(w http.ResponseWriter, r *http.Request, vmid int) {
|
||||
if s.mem == nil {
|
||||
writeErr(w, http.StatusServiceUnavailable, "guest memory resize not configured on this host")
|
||||
return
|
||||
}
|
||||
var req memoryResizeRequest
|
||||
if !decodeBody(w, r, &req) {
|
||||
return
|
||||
}
|
||||
if !s.scopedFromBody(w, req.VMID, vmid, r.URL.Path) {
|
||||
return
|
||||
}
|
||||
if req.MemoryMB <= 0 {
|
||||
writeErr(w, http.StatusBadRequest, "memory_mb must be a positive integer")
|
||||
return
|
||||
}
|
||||
|
||||
// Single-flight: one customer per host; last-write-wins at PVE is not a UX we want.
|
||||
s.memMu.Lock()
|
||||
defer s.memMu.Unlock()
|
||||
|
||||
b, err := s.readMemoryBounds(r.Context(), vmid)
|
||||
if err != nil {
|
||||
s.logger.Error("local-api: guest-memory resize precheck", "vmid", vmid, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "could not read guest memory: "+err.Error())
|
||||
return
|
||||
}
|
||||
target := req.MemoryMB
|
||||
|
||||
// No-op: target == current allocation → success, no SetConfig call.
|
||||
if target == b.allocatedMB {
|
||||
s.logger.Info("local-api: guest-memory resize no-op (unchanged)", "vmid", vmid, "memory_mb", target)
|
||||
writeOK(w, memoryResizeResult{VMID: vmid, OldMB: b.allocatedMB, NewMB: b.allocatedMB, Unchanged: true})
|
||||
return
|
||||
}
|
||||
|
||||
// Ruled refusals — SetConfig MUST NOT run on any of these.
|
||||
switch {
|
||||
case target < b.minMB:
|
||||
s.logger.Warn("local-api: guest-memory resize refused (below_min)", "vmid", vmid, "target_mb", target, "min_mb", b.minMB)
|
||||
writeStatus(w, http.StatusPreconditionFailed, false, b.refusal("below_min"),
|
||||
fmt.Sprintf("requested %d MB is below the minimum %d MB", target, b.minMB))
|
||||
return
|
||||
case target > b.maxMB:
|
||||
s.logger.Warn("local-api: guest-memory resize refused (above_max)", "vmid", vmid, "target_mb", target, "max_mb", b.maxMB)
|
||||
writeStatus(w, http.StatusPreconditionFailed, false, b.refusal("above_max"),
|
||||
fmt.Sprintf("requested %d MB exceeds the maximum %d MB (host reserve %d MB)", target, b.maxMB, hostReserveMB))
|
||||
return
|
||||
case target < b.allocatedMB && target < b.floorMB:
|
||||
// SHRINK below the usage floor — the customer must stop applications first.
|
||||
s.logger.Warn("local-api: guest-memory resize refused (below_usage_floor)",
|
||||
"vmid", vmid, "target_mb", target, "usage_mb", b.usageMB, "floor_mb", b.floorMB)
|
||||
writeStatus(w, http.StatusPreconditionFailed, false, b.refusal("below_usage_floor"),
|
||||
fmt.Sprintf("requested %d MB is too close to current usage %d MB (floor %d MB) — stop applications first", target, b.usageMB, b.floorMB))
|
||||
return
|
||||
}
|
||||
|
||||
// Apply. SetConfig may be synchronous (empty UPID) or return a task to wait on.
|
||||
upid, err := s.mem.SetConfig(r.Context(), vmid, map[string]string{"memory": fmt.Sprintf("%d", target)})
|
||||
if err != nil {
|
||||
s.logger.Error("local-api: guest-memory SetConfig", "vmid", vmid, "target_mb", target, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "memory resize failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if upid != "" {
|
||||
if _, err := s.mem.WaitTask(r.Context(), upid, proxmox.WaitOptions{}); err != nil {
|
||||
s.logger.Error("local-api: guest-memory WaitTask", "vmid", vmid, "upid", upid, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "memory resize task failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Verify: re-read and confirm maxmem reflects the target before claiming success (never trust the
|
||||
// POST 200 — the config could have been rejected at task execution, or applied as pending).
|
||||
st, err := s.mem.GuestStatus(r.Context(), vmid)
|
||||
if err != nil {
|
||||
s.logger.Error("local-api: guest-memory post-apply status", "vmid", vmid, "err", err)
|
||||
writeErr(w, http.StatusBadGateway, "resize applied but verification read failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
if st.MaxMem != target*mib {
|
||||
s.logger.Error("local-api: guest-memory resize NOT reflected after apply",
|
||||
"vmid", vmid, "target_mb", target, "observed_maxmem_bytes", st.MaxMem)
|
||||
writeErr(w, http.StatusBadGateway,
|
||||
fmt.Sprintf("resize did not take effect (guest still reports %d MB) — a reboot may be required", st.MaxMem/mib))
|
||||
return
|
||||
}
|
||||
s.logger.Info("local-api: guest-memory resized", "vmid", vmid, "old_mb", b.allocatedMB, "new_mb", target)
|
||||
writeOK(w, memoryResizeResult{VMID: vmid, OldMB: b.allocatedMB, NewMB: target})
|
||||
}
|
||||
Reference in New Issue
Block a user