From 286a7e3ecfc27fddcc5884bc7e194f08873c2164 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 21 Oct 2026 15:43:18 +0200 Subject: [PATCH] =?UTF-8?q?R-1192:=20every=20guest=20may=20swap=20as=20muc?= =?UTF-8?q?h=20as=20its=20RAM=20(bring-up,=20RAM=20resize,=20once=20at=20s?= =?UTF-8?q?tart)=20=E2=80=94=20the=20measured=208=20GB=20freeze;=20unrelea?= =?UTF-8?q?sed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 18 ++++++ cmd/felhom-agent/main.go | 16 ++++++ internal/localapi/guestmemory.go | 3 +- internal/localapi/guestmemory_test.go | 4 ++ internal/localapi/guestswap.go | 73 +++++++++++++++++++++++++ internal/localapi/guestswap_test.go | 71 ++++++++++++++++++++++++ internal/reconcile/bringup.go | 3 + internal/reconcile/bringup_swap_test.go | 19 +++++++ 8 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 internal/localapi/guestswap.go create mode 100644 internal/localapi/guestswap_test.go create mode 100644 internal/reconcile/bringup_swap_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 006737b..e8f7e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +## Unreleased (2026-10-21) — every guest may swap as much as its RAM (R-1192, the 8 GB freeze) — COMMITTED, NOT RELEASED (the release is the operator's word) + +The freeze of the rehearsal's first afternoon, reproduced and measured on demo-hp (felhom.eu +`audits/fixes-2026-10-21/P4-freeze/`): Proxmox gives an LXC a 512 MB swap cap when nothing sets `swap`, and the agent +never set it. On the 8 GB machine (guest 4096 MB) the first uploads made Immich load its analysis models; the guest hit +its limit with the swap full and could evict only program code — 150 000–600 000 refaults and ~1 GB of disk reads per +10 s, memory „full" pressure 92–95 %, load 155, the dashboard dead for 6½ minutes, no app killed. Raising the swap cap +to 4096 MB live ended it within a minute while the uploads ran on. + +- `reconcile.buildBringUpConfig`: a new guest gets `swap` = its memory. +- `POST /guest/memory` (RAM resize): `swap` moves with `memory` (`localapi.GuestSwapMB`). +- At agent start (one minute in): `localapi.EnsureGuestSwap` raises every LXC in the `felhom` pool whose swap is below + its RAM to it — a live cgroup change, no restart; never lowers a higher swap; one guest's failure is logged and the + others are still done. +- Tests `TestR1192_EnsureGuestSwapRaisesEveryLowSwapToItsRAM`, `TestR1192_BringUpSetsSwapToTheRAM`, the resize test's + new swap assertion — each red-proved (the write removed → `raised 0, want 2`; `swap=""` on both other paths). +- Privilege: `swap` is under `VM.Config.Memory`, which the RAM resize already uses; no role, sudoers or bundle change. + ## v0.161.0 — three fixes (R-921, R-1070, R-1010); two texts made true after 0.160.0 (2026-10-20) — RELEASED on the operator's yes, NOT VOUCHED Released 2026-10-20 (operator yes, the morning answers) with `scripts/release-agent.sh` from a clean clone at `5152123`: diff --git a/cmd/felhom-agent/main.go b/cmd/felhom-agent/main.go index 398f560..12dbaa1 100644 --- a/cmd/felhom-agent/main.go +++ b/cmd/felhom-agent/main.go @@ -936,6 +936,22 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int // network and stays under the hub's 45-minute host_stale (its // alerting.stale_threshold; host_down at 90). On a box without the kernel lane (an older wrapper, a BYO host) the check is refused and logged. go osLeg.KernelAfterBoot(ctx, 0, osupdate.KernelJudge{Wait: osupdate.DefaultKernelJudgeWait}) + // R-1192: every guest may swap as much as its RAM — raised once at start (a live change, no restart). A minute + // after start so the guest has settled; a failure is logged and the next start retries. + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(time.Minute): + } + sctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + if n, err := localapi.EnsureGuestSwap(sctx, px, reconcile.DefaultPool, logger); err != nil { + logger.Warn("guest-swap: check skipped", "err", err) + } else { + logger.Info("guest-swap: checked", "raised", n) + } + }() // Reconcile (slice 4) runs alongside the hub loop, sharing the per-guest queue // (doc 03 §10). At slice 4 the desired-state provider is empty (no hub serving diff --git a/internal/localapi/guestmemory.go b/internal/localapi/guestmemory.go index a511d0f..96fdec6 100644 --- a/internal/localapi/guestmemory.go +++ b/internal/localapi/guestmemory.go @@ -225,7 +225,8 @@ func (s *Server) handleGuestMemoryResize(w http.ResponseWriter, r *http.Request, } // Apply. SetConfig may be synchronous (empty UPID) or return a task to wait on. - upid, err := s.mem.SetConfig(r.Context(), vmid, map[string]string{"memory": fmt.Sprintf("%d", target)}) + // R-1192: the swap moves with the memory (GuestSwapMB) — a grown guest with the old swap cap thrashes the same way. + upid, err := s.mem.SetConfig(r.Context(), vmid, map[string]string{"memory": fmt.Sprintf("%d", target), "swap": fmt.Sprintf("%d", GuestSwapMB(target))}) if err != nil { s.logger.Error("local-api: guest-memory SetConfig", "vmid", vmid, "target_mb", target, "err", err) writeErr(w, http.StatusBadGateway, "memory resize failed: "+err.Error()) diff --git a/internal/localapi/guestmemory_test.go b/internal/localapi/guestmemory_test.go index 8bd721d..a956807 100644 --- a/internal/localapi/guestmemory_test.go +++ b/internal/localapi/guestmemory_test.go @@ -151,6 +151,10 @@ func TestGuestMemory_GrowAndShrink(t *testing.T) { if f.lastParams["memory"] != "12288" { t.Errorf("SetConfig param = %q, want 12288", f.lastParams["memory"]) } + // R-1192: the swap moves with the memory. + if f.lastParams["swap"] != "12288" { + t.Errorf("SetConfig swap = %q, want 12288 (swap = RAM)", f.lastParams["swap"]) + } }) t.Run("shrink above floor", func(t *testing.T) { f := stdFixture() // floor = max(2048, 3000+512) = 3512 diff --git a/internal/localapi/guestswap.go b/internal/localapi/guestswap.go new file mode 100644 index 0000000..a26ad3d --- /dev/null +++ b/internal/localapi/guestswap.go @@ -0,0 +1,73 @@ +package localapi + +import ( + "context" + "fmt" + "log/slog" + + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// ── R-1192 (2026-10-21): the guest may swap as much as its RAM ──────────────────────────────────────────────────── +// +// MEASURED (felhom.eu audits/fixes-2026-10-21/P4-freeze/): on the minimum machine (8 GB host, guest 4096 MB) the +// household's first uploads made Immich load its analysis models (+1.15 GB) on top of 8 apps' 1.75 GB; the guest hit +// its 4096 MB limit with its swap capped at Proxmox's LXC default of 512 MB — and full. The kernel could then evict +// only file pages: ~360 MB was left for the program code of ~25 containers, re-read from disk without end (150 000– +// 600 000 refaults and ~1 GB of reads per 10 s; memory „full" pressure 92–95 %, load 155, the dashboard dead, 0 OOM +// kills). Raising the cap to 4096 MB live ended it in under a minute (memory „full" 96 % → 1.7 %, dashboard back +// 42 s later) while the uploads ran on. The host had 7.4 GB of swap free the whole time. +// +// The agent never set `swap` at all, so every guest had the 512 MB default. Now: +// - a new guest is created with swap = its memory (reconcile.bringup); +// - a RAM resize sets swap with the memory (handleGuestMemoryResize); +// - at agent start, every LXC in the felhom pool whose swap is below its memory is raised to it (this file). +// Raising swap is a live cgroup change (memory.swap.max), no restart. It never lowers a swap someone set higher. + +// GuestSwapMB is the swap a guest of memoryMB gets: the same as its RAM. +func GuestSwapMB(memoryMB int64) int64 { return memoryMB } + +// SwapOps is the Proxmox surface EnsureGuestSwap needs (satisfied by *proxmox.Client). +type SwapOps interface { + Pool(ctx context.Context, name string) (proxmox.PoolInfo, error) + GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error) + SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error) + WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error) +} + +// EnsureGuestSwap raises the swap of every LXC guest in pool to GuestSwapMB(memory) when it is below. Returns how +// many guests were raised. One guest's failure is logged and the others are still handled. +func EnsureGuestSwap(ctx context.Context, ops SwapOps, pool string, logger *slog.Logger) (int, error) { + p, err := ops.Pool(ctx, pool) + if err != nil { + return 0, fmt.Errorf("guest swap: read pool %s: %w", pool, err) + } + raised := 0 + for _, m := range p.Members { + if m.Type != "lxc" || m.VMID <= 0 { + continue + } + cfg, err := ops.GuestConfig(ctx, m.VMID) + if err != nil { + logger.Warn("guest-swap: config read failed (left as it is)", "vmid", m.VMID, "err", err) + continue + } + want := GuestSwapMB(cfg.Memory) + if cfg.Memory <= 0 || cfg.Swap >= want { + logger.Debug("guest-swap: nothing to do", "vmid", m.VMID, "memory_mb", cfg.Memory, "swap_mb", cfg.Swap) + continue + } + upid, err := ops.SetConfig(ctx, m.VMID, map[string]string{"swap": fmt.Sprintf("%d", want)}) + if err == nil && upid != "" { + _, err = ops.WaitTask(ctx, upid, proxmox.WaitOptions{}) + } + if err != nil { + logger.Warn("guest-swap: raise failed (left as it is)", "vmid", m.VMID, "from_mb", cfg.Swap, "to_mb", want, "err", err) + continue + } + raised++ + logger.Info("guest-swap: raised (R-1192 — a guest swap cap below its RAM turned a memory burst into a freeze)", + "vmid", m.VMID, "from_mb", cfg.Swap, "to_mb", want) + } + return raised, nil +} diff --git a/internal/localapi/guestswap_test.go b/internal/localapi/guestswap_test.go new file mode 100644 index 0000000..5a22301 --- /dev/null +++ b/internal/localapi/guestswap_test.go @@ -0,0 +1,71 @@ +package localapi + +import ( + "context" + "errors" + "io" + "log/slog" + "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// R-1192: the freeze measured on the minimum machine (felhom.eu audits/fixes-2026-10-21/P4-freeze/) — a guest whose +// swap is capped below its RAM evicts program code instead of idle memory. EnsureGuestSwap raises every pool guest's +// swap to its RAM, never lowers one, skips non-LXC members, and survives one guest's failure. +// COMPANION RED-PROOF: return 0 before the SetConfig call → „raised 0, want 2". +type fakeSwapOps struct { + cfg map[int]proxmox.GuestConfig + failSet map[int]bool + set map[int]map[string]string +} + +func (f *fakeSwapOps) Pool(context.Context, string) (proxmox.PoolInfo, error) { + return proxmox.PoolInfo{Members: []proxmox.PoolMember{{VMID: 9201, Type: "lxc"}, {VMID: 9202, Type: "lxc"}, {VMID: 300, Type: "qemu"}, + {Type: "storage"}, {VMID: 9203, Type: "lxc"}, {VMID: 9204, Type: "lxc"}}}, nil +} +func (f *fakeSwapOps) GuestConfig(_ context.Context, vmid int) (proxmox.GuestConfig, error) { + c, ok := f.cfg[vmid] + if !ok { + return c, errors.New("no such guest") + } + return c, nil +} +func (f *fakeSwapOps) SetConfig(_ context.Context, vmid int, p map[string]string) (string, error) { + if f.failSet[vmid] { + return "", errors.New("pve said no") + } + if f.set == nil { + f.set = map[int]map[string]string{} + } + f.set[vmid] = p + return "", nil +} +func (f *fakeSwapOps) WaitTask(context.Context, string, proxmox.WaitOptions) (proxmox.TaskStatus, error) { + return proxmox.TaskStatus{}, nil +} + +func TestR1192_EnsureGuestSwapRaisesEveryLowSwapToItsRAM(t *testing.T) { + f := &fakeSwapOps{cfg: map[int]proxmox.GuestConfig{ + 9201: {Memory: 4096, Swap: 512}, // the measured box: raised + 9202: {Memory: 2048, Swap: 4096}, // already higher: never lowered + 9203: {Memory: 6144, Swap: 0}, // none: raised + 9204: {Memory: 4096, Swap: 512}, // the write fails: logged, the others still done + }, failSet: map[int]bool{9204: true}} + n, err := EnsureGuestSwap(context.Background(), f, "felhom", slog.New(slog.NewTextHandler(io.Discard, nil))) + if err != nil { + t.Fatal(err) + } + if n != 2 { + t.Fatalf("raised %d, want 2 (9201, 9203)", n) + } + if f.set[9201]["swap"] != "4096" || f.set[9203]["swap"] != "6144" { + t.Fatalf("writes = %v", f.set) + } + if _, touched := f.set[9202]; touched { + t.Fatal("a swap above the RAM was lowered") + } + if len(f.set[9201]) != 1 { + t.Fatalf("the swap raise wrote more than the swap: %v", f.set[9201]) + } +} diff --git a/internal/reconcile/bringup.go b/internal/reconcile/bringup.go index 1042495..a455a7f 100644 --- a/internal/reconcile/bringup.go +++ b/internal/reconcile/bringup.go @@ -645,6 +645,9 @@ func buildBringUpConfig(spec BringUpSpec, cfg proxmox.GuestConfig) map[string]st } if spec.MemoryMB > 0 { params["memory"] = strconv.Itoa(spec.MemoryMB) + // R-1192: swap = RAM. Left unset, Proxmox gives an LXC 512 MB, and a guest that fills its RAM can then evict + // only program code — the measured freeze (localapi.GuestSwapMB carries the reasoning and the numbers). + params["swap"] = strconv.Itoa(spec.MemoryMB) } for i, m := range spec.Mounts { // backup=1 for data-bearing mounts: extra LXC mountpoints default to backup=0 = EXCLUDED diff --git a/internal/reconcile/bringup_swap_test.go b/internal/reconcile/bringup_swap_test.go new file mode 100644 index 0000000..45ef49d --- /dev/null +++ b/internal/reconcile/bringup_swap_test.go @@ -0,0 +1,19 @@ +package reconcile + +import ( + "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// R-1192: a new guest is created with swap = its RAM (left unset, Proxmox gives an LXC 512 MB — the measured freeze, +// felhom.eu audits/fixes-2026-10-21/P4-freeze/). COMPANION RED-PROOF: drop params["swap"] in buildBringUpConfig. +func TestR1192_BringUpSetsSwapToTheRAM(t *testing.T) { + p := buildBringUpConfig(BringUpSpec{MemoryMB: 4096}, proxmox.GuestConfig{}) + if p["memory"] != "4096" || p["swap"] != "4096" { + t.Fatalf("bring-up params memory=%q swap=%q, want 4096/4096", p["memory"], p["swap"]) + } + if q := buildBringUpConfig(BringUpSpec{}, proxmox.GuestConfig{}); q["swap"] != "" { + t.Fatalf("no memory in the spec → no swap written either, got %q", q["swap"]) + } +}