From 0550b3117e5a6de60aa9dcbb7ef51d7dbc573fe9 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Sun, 14 Jun 2026 09:44:28 +0200 Subject: [PATCH] F1: read cgroup memory limit, not host RAM (restores deploy OOM guard) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /api/system/info reported the Proxmox host's 16GB (the controller container reads host /proc/meminfo with no lxcfs), defeating the deploy memory-headroom hard-block (deploy.go uses GetMemoryMB). readMemInfo now prefers the cgroup memory limit (v2 memory.max / v1 memory.limit_in_bytes; sentinels = unlimited) when finite and below the host total; used = memory.current/usage_in_bytes. Test info_cgroup_test.go (cgroup v2 cap wins, v2 max sentinel, v1 unlimited, v1 finite) — fails on pre-fix code. --- SESSION-2026-06-14-fixbatch.md | 10 ++ .../internal/system/info_cgroup_test.go | 112 ++++++++++++++++++ controller/internal/system/info_linux.go | 84 +++++++++++-- 3 files changed, 198 insertions(+), 8 deletions(-) create mode 100644 SESSION-2026-06-14-fixbatch.md create mode 100644 controller/internal/system/info_cgroup_test.go diff --git a/SESSION-2026-06-14-fixbatch.md b/SESSION-2026-06-14-fixbatch.md new file mode 100644 index 0000000..77afaf3 --- /dev/null +++ b/SESSION-2026-06-14-fixbatch.md @@ -0,0 +1,10 @@ +# SESSION 2026-06-14 — Fix Batch 1 (+F17) → controller v0.61.0 + +Branch: `audit/2026-06-14-live-drive`. Implementing controller-side live-drive fixes per `LIVE-DRIVE-FIXSPEC-2026-06-14.md`. +Exclusions: F9, F20-BUG2, F20-BUG3 (agent/golden — supervised next session). +Operator decisions applied: F17 `.sql` wins; F5 gate route but surface "route unpublished"; F8 0600; F11 warn-and-allow (note: F11 not in this batch's fix list — only F1, F20-BUG1, F4, F6, F7, F8, F5, F17). + +## Progress log +- **t0** — Read CLAUDE.md (build workflow), FIXSPEC, target source files. Setting up. +- **Test approach (documented assumption):** controller code is largely `//go:build linux`; dev host is Windows. Per-commit gate = `CGO_ENABLED=0 GOOS=linux go build ./... && go vet ./...` locally (compiles + type-checks code AND `_test.go`). Authoritative `go test ./...` runs on the Linux build server (192.168.0.180) against the pushed branch BEFORE the image build/deploy. Windows cannot execute linux test binaries, so this is the faithful interpretation of the per-commit green gate. +- **t1 — F1 (cgroup memory)** DONE. `internal/system/info_linux.go`: `readMemInfo` now prefers the cgroup memory LIMIT (v2 `memory.max`, v1 `memory.limit_in_bytes`, "max"/near-uint64-max = unlimited→fallback) when finite and below the host `/proc/meminfo` total; used = `memory.current`/`usage_in_bytes`. Restores the deploy OOM guard (deploy.go:162-185 reads GetMemoryMB). Test `info_cgroup_test.go` (4 cases) — fails on pre-fix code (ignored cgroup). GOOS=linux build+vet OK. diff --git a/controller/internal/system/info_cgroup_test.go b/controller/internal/system/info_cgroup_test.go new file mode 100644 index 0000000..c22591f --- /dev/null +++ b/controller/internal/system/info_cgroup_test.go @@ -0,0 +1,112 @@ +//go:build linux + +package system + +import ( + "os" + "path/filepath" + "testing" +) + +// TestReadMemInfoUsesCgroupV2Limit asserts F1: when a cgroup v2 memory.max caps the container well +// below the host /proc/meminfo total, readMemInfo reports the cgroup cap (the guest's real ceiling), +// not the host RAM. On the pre-fix code this test fails because readMemInfo ignored cgroup entirely. +func TestReadMemInfoUsesCgroupV2Limit(t *testing.T) { + dir := t.TempDir() + // 2 GiB limit, 512 MiB current usage. + const twoGiB = uint64(2 * 1024 * 1024 * 1024) + const halfGiB = uint64(512 * 1024 * 1024) + mustWriteFile(t, filepath.Join(dir, "memory.max"), []byte(itoa(twoGiB))) + mustWriteFile(t, filepath.Join(dir, "memory.current"), []byte(itoa(halfGiB))) + + old := cgroupRoot + cgroupRoot = dir + defer func() { cgroupRoot = old }() + + var info SystemInfo + readMemInfo(&info) + + // Host /proc/meminfo total is whatever the test machine has; the cgroup cap must win when smaller. + if info.TotalMemMB != 2048 { + t.Fatalf("TotalMemMB = %d, want 2048 (cgroup cap), not host RAM", info.TotalMemMB) + } + if info.UsedMemMB != 512 { + t.Fatalf("UsedMemMB = %d, want 512 (memory.current)", info.UsedMemMB) + } + if info.AvailMemMB != 1536 { + t.Fatalf("AvailMemMB = %d, want 1536", info.AvailMemMB) + } + if info.MemPercent < 24 || info.MemPercent > 26 { + t.Fatalf("MemPercent = %.1f, want ~25", info.MemPercent) + } +} + +// TestReadMemInfoCgroupMaxIsUnlimited asserts that a v2 "max" sentinel does NOT override /proc/meminfo +// (an uncapped container keeps the host view rather than a bogus 0). +func TestReadMemInfoCgroupMaxIsUnlimited(t *testing.T) { + dir := t.TempDir() + mustWriteFile(t, filepath.Join(dir, "memory.max"), []byte("max")) + + old := cgroupRoot + cgroupRoot = dir + defer func() { cgroupRoot = old }() + + var info SystemInfo + readMemInfo(&info) + + if info.TotalMemMB == 0 { + t.Fatalf("TotalMemMB = 0 with an unlimited cgroup; expected the /proc/meminfo host total") + } +} + +// TestReadCgroupMemLimitV1Unlimited asserts the v1 near-uint64-max sentinel is treated as unlimited. +func TestReadCgroupMemLimitV1Unlimited(t *testing.T) { + dir := t.TempDir() + memDir := filepath.Join(dir, "memory") + if err := os.MkdirAll(memDir, 0o755); err != nil { + t.Fatal(err) + } + // Typical v1 "unlimited" value. + mustWriteFile(t, filepath.Join(memDir, "memory.limit_in_bytes"), []byte("9223372036854771712")) + + if _, ok := readCgroupMemLimitMB(dir); ok { + t.Fatalf("readCgroupMemLimitMB treated the v1 unlimited sentinel as a real limit") + } +} + +// TestReadCgroupMemLimitV1Real asserts a finite v1 limit is read. +func TestReadCgroupMemLimitV1Real(t *testing.T) { + dir := t.TempDir() + memDir := filepath.Join(dir, "memory") + if err := os.MkdirAll(memDir, 0o755); err != nil { + t.Fatal(err) + } + const oneGiB = uint64(1024 * 1024 * 1024) + mustWriteFile(t, filepath.Join(memDir, "memory.limit_in_bytes"), []byte(itoa(oneGiB))) + + mb, ok := readCgroupMemLimitMB(dir) + if !ok || mb != 1024 { + t.Fatalf("readCgroupMemLimitMB = (%d, %v), want (1024, true)", mb, ok) + } +} + +func mustWriteFile(t *testing.T, path string, data []byte) { + t.Helper() + if err := os.WriteFile(path, data, 0o644); err != nil { + t.Fatal(err) + } +} + +func itoa(v uint64) string { + if v == 0 { + return "0" + } + var buf [20]byte + i := len(buf) + for v > 0 { + i-- + buf[i] = byte('0' + v%10) + v /= 10 + } + return string(buf[i:]) +} diff --git a/controller/internal/system/info_linux.go b/controller/internal/system/info_linux.go index 1b0091d..836bb9b 100644 --- a/controller/internal/system/info_linux.go +++ b/controller/internal/system/info_linux.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "syscall" "time" @@ -77,6 +78,9 @@ func GetMemoryMB() (totalMB, usedMB int, err error) { return int(info.TotalMemMB), int(info.UsedMemMB), nil } +// cgroupRoot is the cgroup mount point. Overridable in tests. +var cgroupRoot = "/sys/fs/cgroup" + func readMemInfo(info *SystemInfo) { f, err := os.Open("/proc/meminfo") if err != nil { @@ -100,16 +104,80 @@ func readMemInfo(info *SystemInfo) { } } - if totalKB > 0 { - info.TotalMemMB = totalKB / 1024 - info.AvailMemMB = availKB / 1024 - info.UsedMemMB = info.TotalMemMB - info.AvailMemMB - info.MemPercent = float64(info.UsedMemMB) / float64(info.TotalMemMB) * 100 - debugf("[DEBUG] [system] readMemInfo: totalKB=%d availKB=%d → total=%dMB avail=%dMB used=%dMB (%.1f%%)", - totalKB, availKB, info.TotalMemMB, info.AvailMemMB, info.UsedMemMB, info.MemPercent) - } else { + if totalKB == 0 { debugf("[DEBUG] [system] readMemInfo: could not parse MemTotal from /proc/meminfo") + return } + + info.TotalMemMB = totalKB / 1024 + info.AvailMemMB = availKB / 1024 + info.UsedMemMB = info.TotalMemMB - info.AvailMemMB + + // F1: the controller runs as a Docker container inside an LXC. /proc/meminfo reports the HOST's + // RAM (no lxcfs in the container), which massively overstates the guest's real ceiling and defeats + // the deploy memory-headroom guard. Prefer the cgroup memory LIMIT when it is finite and below the + // host total — that is the amount this guest can actually use. Fall back to /proc/meminfo otherwise. + if limitMB, ok := readCgroupMemLimitMB(cgroupRoot); ok && limitMB > 0 && limitMB < info.TotalMemMB { + info.TotalMemMB = limitMB + if curMB, okC := readCgroupMemCurrentMB(cgroupRoot); okC && curMB <= limitMB { + info.UsedMemMB = curMB + } else if info.UsedMemMB > limitMB { + info.UsedMemMB = limitMB + } + info.AvailMemMB = info.TotalMemMB - info.UsedMemMB + debugf("[DEBUG] [system] readMemInfo: using cgroup limit=%dMB (host total was %dKB) → used=%dMB avail=%dMB", + limitMB, totalKB, info.UsedMemMB, info.AvailMemMB) + } + + if info.TotalMemMB > 0 { + info.MemPercent = float64(info.UsedMemMB) / float64(info.TotalMemMB) * 100 + } + debugf("[DEBUG] [system] readMemInfo: totalKB=%d availKB=%d → total=%dMB avail=%dMB used=%dMB (%.1f%%)", + totalKB, availKB, info.TotalMemMB, info.AvailMemMB, info.UsedMemMB, info.MemPercent) +} + +// readCgroupMemLimitMB returns the cgroup memory limit in MB. It tries cgroup v2 (memory.max) first, +// then v1 (memory/memory.limit_in_bytes). A sentinel ("max" on v2, or a near-uint64-max value on v1) +// means "unlimited" → ok=false so the caller keeps the /proc/meminfo value. +func readCgroupMemLimitMB(root string) (mb uint64, ok bool) { + // cgroup v2 + if b, err := os.ReadFile(filepath.Join(root, "memory.max")); err == nil { + s := strings.TrimSpace(string(b)) + if s == "max" { + return 0, false + } + if v, err := strconv.ParseUint(s, 10, 64); err == nil && v > 0 { + return v / (1024 * 1024), true + } + } + // cgroup v1 + if b, err := os.ReadFile(filepath.Join(root, "memory", "memory.limit_in_bytes")); err == nil { + s := strings.TrimSpace(string(b)) + if v, err := strconv.ParseUint(s, 10, 64); err == nil && v > 0 { + // v1 "unlimited" is a huge page-aligned value near uint64 max; treat >= 1 PiB as unlimited. + if v >= (1 << 50) { + return 0, false + } + return v / (1024 * 1024), true + } + } + return 0, false +} + +// readCgroupMemCurrentMB returns the cgroup current memory usage in MB (v2 memory.current, v1 +// memory/memory.usage_in_bytes). ok=false if unreadable. +func readCgroupMemCurrentMB(root string) (mb uint64, ok bool) { + for _, p := range []string{ + filepath.Join(root, "memory.current"), + filepath.Join(root, "memory", "memory.usage_in_bytes"), + } { + if b, err := os.ReadFile(p); err == nil { + if v, err := strconv.ParseUint(strings.TrimSpace(string(b)), 10, 64); err == nil { + return v / (1024 * 1024), true + } + } + } + return 0, false } // parseMemLine extracts the kB value from a /proc/meminfo line like "MemTotal: 16384000 kB"