diff --git a/controller/internal/api/router.go b/controller/internal/api/router.go index 1920a1b..7f90d2a 100644 --- a/controller/internal/api/router.go +++ b/controller/internal/api/router.go @@ -719,6 +719,20 @@ func (r *Router) triggerSync(w http.ResponseWriter, _ *http.Request) { func (r *Router) systemInfo(w http.ResponseWriter, _ *http.Request) { info := system.GetInfo(r.cfg.Paths.HDDPath, r.cpuCollector) + // F1: GetInfo now reports the guest RAM cap (from the Docker daemon) as TotalMemMB, but the guest-wide + // "used" is not observable from the container. Report the controller's accurate committed-app memory + // (sum of running apps' mem requests) as used — a meaningful "allocated of cap" figure for the UI. + if r.stackMgr != nil && info.TotalMemMB > 0 { + if reqMB, _ := r.stackMgr.CommittedMemory(); reqMB >= 0 { + used := uint64(reqMB) + if used > info.TotalMemMB { + used = info.TotalMemMB + } + info.UsedMemMB = used + info.AvailMemMB = info.TotalMemMB - used + info.MemPercent = float64(used) / float64(info.TotalMemMB) * 100 + } + } syncStatus := r.syncer.Status() data := map[string]interface{}{ "system": info, diff --git a/controller/internal/stacks/deploy.go b/controller/internal/stacks/deploy.go index 6bc64c0..08341ec 100644 --- a/controller/internal/stacks/deploy.go +++ b/controller/internal/stacks/deploy.go @@ -160,16 +160,28 @@ func (m *Manager) DeployStack(req DeployRequest) (string, error) { var deployWarning string reservedMB := m.cfg.System.ReservedMemoryMB totalMB, usedMB, memErr := system.GetMemoryMB() + // F1: the controller container cannot read the guest's RAM cap from /proc (no lxcfs) or its own + // cgroup (the cap is on the LXC ancestor). Prefer the guest cap from the Docker daemon (runs in the + // LXC). And use the controller's OWN committed-memory accounting for "used" — accurate and cheap — + // rather than host /proc RSS, which is unobservable-per-guest and would otherwise make this guard + // either never fire (host total) or always fire (host used > guest cap). + if gt, ok := system.GuestMemTotalMB(); ok && gt > 0 { + totalMB = gt + memErr = nil + } + if committedReqMB, _ := m.CommittedMemory(); committedReqMB > 0 || memErr == nil { + usedMB = committedReqMB + } if memErr != nil { m.logger.Printf("[WARN] [stacks] Cannot read system memory: %v — skipping memory check", memErr) } else { usableMB := totalMB - reservedMB newReqMB := ParseMemoryMB(meta.Resources.MemRequest) - m.logger.Printf("[INFO] [stacks] Memory check: total=%dMB, reserved=%dMB, usable=%dMB, real_used=%dMB, new_req=%dMB, remaining=%dMB", + m.logger.Printf("[INFO] [stacks] Memory check: total=%dMB, reserved=%dMB, usable=%dMB, committed_used=%dMB, new_req=%dMB, remaining=%dMB", totalMB, reservedMB, usableMB, usedMB, newReqMB, usableMB-usedMB-newReqMB) - // Hard block: real used + new request exceeds usable memory + // Hard block: committed + new request exceeds usable memory if newReqMB > 0 && usedMB+newReqMB > usableMB { clearDeploying() return "", fmt.Errorf( diff --git a/controller/internal/system/info_cgroup_test.go b/controller/internal/system/info_cgroup_test.go index c22591f..a3c954d 100644 --- a/controller/internal/system/info_cgroup_test.go +++ b/controller/internal/system/info_cgroup_test.go @@ -13,11 +13,8 @@ import ( // not the host RAM. On the pre-fix code this test fails because readMemInfo ignored cgroup entirely. func TestReadMemInfoUsesCgroupV2Limit(t *testing.T) { dir := t.TempDir() - // 2 GiB limit, 512 MiB current usage. const twoGiB = uint64(2 * 1024 * 1024 * 1024) - const halfGiB = uint64(512 * 1024 * 1024) mustWriteFile(t, filepath.Join(dir, "memory.max"), []byte(itoa(twoGiB))) - mustWriteFile(t, filepath.Join(dir, "memory.current"), []byte(itoa(halfGiB))) old := cgroupRoot cgroupRoot = dir @@ -30,14 +27,48 @@ func TestReadMemInfoUsesCgroupV2Limit(t *testing.T) { if info.TotalMemMB != 2048 { t.Fatalf("TotalMemMB = %d, want 2048 (cgroup cap), not host RAM", info.TotalMemMB) } - if info.UsedMemMB != 512 { - t.Fatalf("UsedMemMB = %d, want 512 (memory.current)", info.UsedMemMB) + // Used is a scaled estimate (the container cannot read guest-wide RSS); just assert it is sane. + if info.UsedMemMB > info.TotalMemMB { + t.Fatalf("UsedMemMB = %d exceeds TotalMemMB = %d", info.UsedMemMB, info.TotalMemMB) } - if info.AvailMemMB != 1536 { - t.Fatalf("AvailMemMB = %d, want 1536", info.AvailMemMB) + if info.AvailMemMB != info.TotalMemMB-info.UsedMemMB { + t.Fatalf("AvailMemMB inconsistent: %d != %d-%d", info.AvailMemMB, info.TotalMemMB, info.UsedMemMB) } - if info.MemPercent < 24 || info.MemPercent > 26 { - t.Fatalf("MemPercent = %.1f, want ~25", info.MemPercent) +} + +// TestReadMemInfoUsesDockerInfoWhenCgroupUnlimited asserts the NESTED-LXC case (the real demo): the +// container's own cgroup is unlimited ("max"), so the guest cap must come from `docker info` MemTotal. +// Pre-fix (and the cgroup-only attempt) reports the host RAM here. +func TestReadMemInfoUsesDockerInfoWhenCgroupUnlimited(t *testing.T) { + dir := t.TempDir() + mustWriteFile(t, filepath.Join(dir, "memory.max"), []byte("max")) // container cgroup unlimited + old := cgroupRoot + cgroupRoot = dir + defer func() { cgroupRoot = old }() + oldFn := dockerMemTotalFn + dockerMemTotalFn = func() (uint64, bool) { return 2048, true } + defer func() { dockerMemTotalFn = oldFn }() + + var info SystemInfo + readMemInfo(&info) + if info.TotalMemMB != 2048 { + t.Fatalf("TotalMemMB = %d, want 2048 (docker-info guest cap)", info.TotalMemMB) + } +} + +// TestGuestMemTotalMB_DockerInfoFallback asserts GuestMemTotalMB (used by the deploy guard) falls back +// to docker-info when no cgroup limit is present. +func TestGuestMemTotalMB_DockerInfoFallback(t *testing.T) { + dir := t.TempDir() // no cgroup files → no limit + old := cgroupRoot + cgroupRoot = dir + defer func() { cgroupRoot = old }() + oldFn := dockerMemTotalFn + dockerMemTotalFn = func() (uint64, bool) { return 2048, true } + defer func() { dockerMemTotalFn = oldFn }() + + if v, ok := GuestMemTotalMB(); !ok || v != 2048 { + t.Fatalf("GuestMemTotalMB = (%d, %v), want (2048, true)", v, ok) } } diff --git a/controller/internal/system/info_linux.go b/controller/internal/system/info_linux.go index 836bb9b..af90414 100644 --- a/controller/internal/system/info_linux.go +++ b/controller/internal/system/info_linux.go @@ -4,8 +4,10 @@ package system import ( "bufio" + "context" "fmt" "os" + "os/exec" "path/filepath" "sort" "strconv" @@ -113,20 +115,38 @@ func readMemInfo(info *SystemInfo) { info.AvailMemMB = availKB / 1024 info.UsedMemMB = info.TotalMemMB - info.AvailMemMB - // F1: the controller runs as a Docker container inside an LXC. /proc/meminfo reports the HOST's - // RAM (no lxcfs in the container), which massively overstates the guest's real ceiling and defeats - // the deploy memory-headroom guard. Prefer the cgroup memory LIMIT when it is finite and below the - // host total — that is the amount this guest can actually use. Fall back to /proc/meminfo otherwise. - if limitMB, ok := readCgroupMemLimitMB(cgroupRoot); ok && limitMB > 0 && limitMB < info.TotalMemMB { - info.TotalMemMB = limitMB - if curMB, okC := readCgroupMemCurrentMB(cgroupRoot); okC && curMB <= limitMB { - info.UsedMemMB = curMB - } else if info.UsedMemMB > limitMB { - info.UsedMemMB = limitMB + // F1: the controller runs as a Docker container inside an LXC. /proc/meminfo reports the HOST's RAM + // (no lxcfs in the container) and the container's OWN cgroup is unlimited (the 2GB cap lives on the + // LXC, an ancestor hidden from the container), so the reported total massively overstates the guest's + // real ceiling and defeats the deploy memory-headroom guard. Determine the true guest cap from, in + // order: the container's cgroup limit (correct when Docker sets -m, e.g. non-nested deploys), else + // `docker info` MemTotal (dockerd runs IN the LXC and reports the guest's lxcfs-backed RAM — the + // accurate cap in the nested-LXC case). The instantaneous guest-wide RSS is NOT observable from the + // container, so when we override the cap we scale the host's used-fraction onto it as an estimate for + // display; the CAP itself (what the headroom math depends on) is accurate. The deploy guard uses the + // controller's own committed-memory accounting for "used", so safety does not rely on this estimate. + capMB := uint64(0) + if v, ok := readCgroupMemLimitMB(cgroupRoot); ok && v > 0 && v < info.TotalMemMB { + capMB = v + } + if capMB == 0 { + if v, ok := guestMemTotalMB(); ok && v > 0 && v < info.TotalMemMB { + capMB = v } + } + if capMB > 0 && capMB < info.TotalMemMB { + frac := 0.0 + if info.TotalMemMB > 0 { + frac = float64(info.UsedMemMB) / float64(info.TotalMemMB) + } + info.TotalMemMB = capMB + // Scaled host-pressure estimate (the container can't read guest-wide RSS). The /api/system/info + // handler overrides this with the controller's committed-app memory for an accurate figure; this + // estimate covers the other GetInfo callers (monitoring) without alarming at ~100%. + info.UsedMemMB = uint64(float64(capMB) * frac) info.AvailMemMB = info.TotalMemMB - info.UsedMemMB - debugf("[DEBUG] [system] readMemInfo: using cgroup limit=%dMB (host total was %dKB) → used=%dMB avail=%dMB", - limitMB, totalKB, info.UsedMemMB, info.AvailMemMB) + debugf("[DEBUG] [system] readMemInfo: guest cap=%dMB (host total was %dKB) → used≈%dMB avail≈%dMB", + capMB, totalKB, info.UsedMemMB, info.AvailMemMB) } if info.TotalMemMB > 0 { @@ -136,6 +156,43 @@ func readMemInfo(info *SystemInfo) { totalKB, availKB, info.TotalMemMB, info.AvailMemMB, info.UsedMemMB, info.MemPercent) } +// guestMemTotalMB returns the guest's total RAM (MB) as reported by the Docker daemon. The daemon runs +// inside the LXC, so `docker info` MemTotal reflects the guest's lxcfs-backed /proc/meminfo (the real +// cap) — unlike the container's own /proc/meminfo, which shows the Proxmox host's RAM. Overridable in +// tests via dockerMemTotalFn. +func guestMemTotalMB() (uint64, bool) { + if dockerMemTotalFn != nil { + return dockerMemTotalFn() + } + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, "docker", "info", "--format", "{{.MemTotal}}").Output() + if err != nil { + return 0, false + } + bytesVal, err := strconv.ParseUint(strings.TrimSpace(string(out)), 10, 64) + if err != nil || bytesVal == 0 { + return 0, false + } + return bytesVal / (1024 * 1024), true +} + +// dockerMemTotalFn lets tests stub the docker-info read. +var dockerMemTotalFn func() (uint64, bool) + +// GuestMemTotalMB returns the guest's memory cap in MB (docker-info MemTotal), preferring the cgroup +// limit when finite. ok=false if neither is determinable. The deploy memory guard uses this as the +// accurate denominator (the controller container cannot read the guest cap from /proc — no lxcfs). +func GuestMemTotalMB() (int, bool) { + if v, ok := readCgroupMemLimitMB(cgroupRoot); ok && v > 0 { + return int(v), true + } + if v, ok := guestMemTotalMB(); ok && v > 0 { + return int(v), true + } + return 0, false +} + // readCgroupMemLimitMB returns the cgroup memory limit in MB. It tries cgroup v2 (memory.max) first, // then v1 (memory/memory.limit_in_bytes). A sentinel ("max" on v2, or a near-uint64-max value on v1) // means "unlimited" → ok=false so the caller keeps the /proc/meminfo value. diff --git a/controller/internal/system/info_other.go b/controller/internal/system/info_other.go index feccb79..b114d88 100644 --- a/controller/internal/system/info_other.go +++ b/controller/internal/system/info_other.go @@ -18,3 +18,8 @@ func GetTotalMemoryMB() (int, error) { func GetMemoryMB() (totalMB, usedMB int, err error) { return 0, 0, fmt.Errorf("/proc/meminfo not available on this platform") } + +// GuestMemTotalMB is not determinable on non-Linux platforms. +func GuestMemTotalMB() (int, bool) { + return 0, false +}