R-1192: every guest may swap as much as its RAM (bring-up, RAM resize, once at start) — the measured 8 GB freeze; unreleased

CHANGELOG.md

@@ -1,3 +1,21 @@
+## Unreleased (2026-10-21) — every guest may swap as much as its RAM (R-1192, the 8 GB freeze) — COMMITTED, NOT RELEASED (the release is the operator's word)
+
+The freeze of the rehearsal's first afternoon, reproduced and measured on demo-hp (felhom.eu
+`audits/fixes-2026-10-21/P4-freeze/`): Proxmox gives an LXC a 512 MB swap cap when nothing sets `swap`, and the agent
+never set it. On the 8 GB machine (guest 4096 MB) the first uploads made Immich load its analysis models; the guest hit
+its limit with the swap full and could evict only program code — 150 000–600 000 refaults and ~1 GB of disk reads per
+10 s, memory „full" pressure 92–95 %, load 155, the dashboard dead for 6½ minutes, no app killed. Raising the swap cap
+to 4096 MB live ended it within a minute while the uploads ran on.
+
+- `reconcile.buildBringUpConfig`: a new guest gets `swap` = its memory.
+- `POST /guest/memory` (RAM resize): `swap` moves with `memory` (`localapi.GuestSwapMB`).
+- At agent start (one minute in): `localapi.EnsureGuestSwap` raises every LXC in the `felhom` pool whose swap is below
+  its RAM to it — a live cgroup change, no restart; never lowers a higher swap; one guest's failure is logged and the
+  others are still done.
+- Tests `TestR1192_EnsureGuestSwapRaisesEveryLowSwapToItsRAM`, `TestR1192_BringUpSetsSwapToTheRAM`, the resize test's
+  new swap assertion — each red-proved (the write removed → `raised 0, want 2`; `swap=""` on both other paths).
+- Privilege: `swap` is under `VM.Config.Memory`, which the RAM resize already uses; no role, sudoers or bundle change.
+
 ## v0.161.0 — three fixes (R-921, R-1070, R-1010); two texts made true after 0.160.0 (2026-10-20) — RELEASED on the operator's yes, NOT VOUCHED
 
 Released 2026-10-20 (operator yes, the morning answers) with `scripts/release-agent.sh` from a clean clone at `5152123`:

cmd/felhom-agent/main.go

@@ -936,6 +936,22 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
 	// network and stays under the hub's 45-minute host_stale (its
 	// alerting.stale_threshold; host_down at 90). On a box without the kernel lane (an older wrapper, a BYO host) the check is refused and logged.
 	go osLeg.KernelAfterBoot(ctx, 0, osupdate.KernelJudge{Wait: osupdate.DefaultKernelJudgeWait})
+	// R-1192: every guest may swap as much as its RAM — raised once at start (a live change, no restart). A minute
+	// after start so the guest has settled; a failure is logged and the next start retries.
+	go func() {
+		select {
+		case <-ctx.Done():
+			return
+		case <-time.After(time.Minute):
+		}
+		sctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+		defer cancel()
+		if n, err := localapi.EnsureGuestSwap(sctx, px, reconcile.DefaultPool, logger); err != nil {
+			logger.Warn("guest-swap: check skipped", "err", err)
+		} else {
+			logger.Info("guest-swap: checked", "raised", n)
+		}
+	}()
 
 	// Reconcile (slice 4) runs alongside the hub loop, sharing the per-guest queue
 	// (doc 03 §10). At slice 4 the desired-state provider is empty (no hub serving

internal/localapi/guestmemory.go

@@ -225,7 +225,8 @@ func (s *Server) handleGuestMemoryResize(w http.ResponseWriter, r *http.Request,
 	}
 
 	// Apply. SetConfig may be synchronous (empty UPID) or return a task to wait on.
-	upid, err := s.mem.SetConfig(r.Context(), vmid, map[string]string{"memory": fmt.Sprintf("%d", target)})
+	// R-1192: the swap moves with the memory (GuestSwapMB) — a grown guest with the old swap cap thrashes the same way.
+	upid, err := s.mem.SetConfig(r.Context(), vmid, map[string]string{"memory": fmt.Sprintf("%d", target), "swap": fmt.Sprintf("%d", GuestSwapMB(target))})
 	if err != nil {
 		s.logger.Error("local-api: guest-memory SetConfig", "vmid", vmid, "target_mb", target, "err", err)
 		writeErr(w, http.StatusBadGateway, "memory resize failed: "+err.Error())

internal/localapi/guestmemory_test.go

@@ -151,6 +151,10 @@ func TestGuestMemory_GrowAndShrink(t *testing.T) {
 		if f.lastParams["memory"] != "12288" {
 			t.Errorf("SetConfig param = %q, want 12288", f.lastParams["memory"])
 		}
+		// R-1192: the swap moves with the memory.
+		if f.lastParams["swap"] != "12288" {
+			t.Errorf("SetConfig swap = %q, want 12288 (swap = RAM)", f.lastParams["swap"])
+		}
 	})
 	t.Run("shrink above floor", func(t *testing.T) {
 		f := stdFixture() // floor = max(2048, 3000+512) = 3512

internal/localapi/guestswap.go

@@ -0,0 +1,73 @@
+package localapi
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// ── R-1192 (2026-10-21): the guest may swap as much as its RAM ────────────────────────────────────────────────────
+//
+// MEASURED (felhom.eu audits/fixes-2026-10-21/P4-freeze/): on the minimum machine (8 GB host, guest 4096 MB) the
+// household's first uploads made Immich load its analysis models (+1.15 GB) on top of 8 apps' 1.75 GB; the guest hit
+// its 4096 MB limit with its swap capped at Proxmox's LXC default of 512 MB — and full. The kernel could then evict
+// only file pages: ~360 MB was left for the program code of ~25 containers, re-read from disk without end (150 000–
+// 600 000 refaults and ~1 GB of reads per 10 s; memory „full" pressure 92–95 %, load 155, the dashboard dead, 0 OOM
+// kills). Raising the cap to 4096 MB live ended it in under a minute (memory „full" 96 % → 1.7 %, dashboard back
+// 42 s later) while the uploads ran on. The host had 7.4 GB of swap free the whole time.
+//
+// The agent never set `swap` at all, so every guest had the 512 MB default. Now:
+//   - a new guest is created with swap = its memory (reconcile.bringup);
+//   - a RAM resize sets swap with the memory (handleGuestMemoryResize);
+//   - at agent start, every LXC in the felhom pool whose swap is below its memory is raised to it (this file).
+// Raising swap is a live cgroup change (memory.swap.max), no restart. It never lowers a swap someone set higher.
+
+// GuestSwapMB is the swap a guest of memoryMB gets: the same as its RAM.
+func GuestSwapMB(memoryMB int64) int64 { return memoryMB }
+
+// SwapOps is the Proxmox surface EnsureGuestSwap needs (satisfied by *proxmox.Client).
+type SwapOps interface {
+	Pool(ctx context.Context, name string) (proxmox.PoolInfo, error)
+	GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
+	SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error)
+	WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
+}
+
+// EnsureGuestSwap raises the swap of every LXC guest in pool to GuestSwapMB(memory) when it is below. Returns how
+// many guests were raised. One guest's failure is logged and the others are still handled.
+func EnsureGuestSwap(ctx context.Context, ops SwapOps, pool string, logger *slog.Logger) (int, error) {
+	p, err := ops.Pool(ctx, pool)
+	if err != nil {
+		return 0, fmt.Errorf("guest swap: read pool %s: %w", pool, err)
+	}
+	raised := 0
+	for _, m := range p.Members {
+		if m.Type != "lxc" || m.VMID <= 0 {
+			continue
+		}
+		cfg, err := ops.GuestConfig(ctx, m.VMID)
+		if err != nil {
+			logger.Warn("guest-swap: config read failed (left as it is)", "vmid", m.VMID, "err", err)
+			continue
+		}
+		want := GuestSwapMB(cfg.Memory)
+		if cfg.Memory <= 0 || cfg.Swap >= want {
+			logger.Debug("guest-swap: nothing to do", "vmid", m.VMID, "memory_mb", cfg.Memory, "swap_mb", cfg.Swap)
+			continue
+		}
+		upid, err := ops.SetConfig(ctx, m.VMID, map[string]string{"swap": fmt.Sprintf("%d", want)})
+		if err == nil && upid != "" {
+			_, err = ops.WaitTask(ctx, upid, proxmox.WaitOptions{})
+		}
+		if err != nil {
+			logger.Warn("guest-swap: raise failed (left as it is)", "vmid", m.VMID, "from_mb", cfg.Swap, "to_mb", want, "err", err)
+			continue
+		}
+		raised++
+		logger.Info("guest-swap: raised (R-1192 — a guest swap cap below its RAM turned a memory burst into a freeze)",
+			"vmid", m.VMID, "from_mb", cfg.Swap, "to_mb", want)
+	}
+	return raised, nil
+}

internal/localapi/guestswap_test.go

@@ -0,0 +1,71 @@
+package localapi
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// R-1192: the freeze measured on the minimum machine (felhom.eu audits/fixes-2026-10-21/P4-freeze/) — a guest whose
+// swap is capped below its RAM evicts program code instead of idle memory. EnsureGuestSwap raises every pool guest's
+// swap to its RAM, never lowers one, skips non-LXC members, and survives one guest's failure.
+// COMPANION RED-PROOF: return 0 before the SetConfig call → „raised 0, want 2".
+type fakeSwapOps struct {
+	cfg     map[int]proxmox.GuestConfig
+	failSet map[int]bool
+	set     map[int]map[string]string
+}
+
+func (f *fakeSwapOps) Pool(context.Context, string) (proxmox.PoolInfo, error) {
+	return proxmox.PoolInfo{Members: []proxmox.PoolMember{{VMID: 9201, Type: "lxc"}, {VMID: 9202, Type: "lxc"}, {VMID: 300, Type: "qemu"},
+		{Type: "storage"}, {VMID: 9203, Type: "lxc"}, {VMID: 9204, Type: "lxc"}}}, nil
+}
+func (f *fakeSwapOps) GuestConfig(_ context.Context, vmid int) (proxmox.GuestConfig, error) {
+	c, ok := f.cfg[vmid]
+	if !ok {
+		return c, errors.New("no such guest")
+	}
+	return c, nil
+}
+func (f *fakeSwapOps) SetConfig(_ context.Context, vmid int, p map[string]string) (string, error) {
+	if f.failSet[vmid] {
+		return "", errors.New("pve said no")
+	}
+	if f.set == nil {
+		f.set = map[int]map[string]string{}
+	}
+	f.set[vmid] = p
+	return "", nil
+}
+func (f *fakeSwapOps) WaitTask(context.Context, string, proxmox.WaitOptions) (proxmox.TaskStatus, error) {
+	return proxmox.TaskStatus{}, nil
+}
+
+func TestR1192_EnsureGuestSwapRaisesEveryLowSwapToItsRAM(t *testing.T) {
+	f := &fakeSwapOps{cfg: map[int]proxmox.GuestConfig{
+		9201: {Memory: 4096, Swap: 512},  // the measured box: raised
+		9202: {Memory: 2048, Swap: 4096}, // already higher: never lowered
+		9203: {Memory: 6144, Swap: 0},    // none: raised
+		9204: {Memory: 4096, Swap: 512},  // the write fails: logged, the others still done
+	}, failSet: map[int]bool{9204: true}}
+	n, err := EnsureGuestSwap(context.Background(), f, "felhom", slog.New(slog.NewTextHandler(io.Discard, nil)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if n != 2 {
+		t.Fatalf("raised %d, want 2 (9201, 9203)", n)
+	}
+	if f.set[9201]["swap"] != "4096" || f.set[9203]["swap"] != "6144" {
+		t.Fatalf("writes = %v", f.set)
+	}
+	if _, touched := f.set[9202]; touched {
+		t.Fatal("a swap above the RAM was lowered")
+	}
+	if len(f.set[9201]) != 1 {
+		t.Fatalf("the swap raise wrote more than the swap: %v", f.set[9201])
+	}
+}

internal/reconcile/bringup.go

@@ -645,6 +645,9 @@ func buildBringUpConfig(spec BringUpSpec, cfg proxmox.GuestConfig) map[string]st
 	}
 	if spec.MemoryMB > 0 {
 		params["memory"] = strconv.Itoa(spec.MemoryMB)
+		// R-1192: swap = RAM. Left unset, Proxmox gives an LXC 512 MB, and a guest that fills its RAM can then evict
+		// only program code — the measured freeze (localapi.GuestSwapMB carries the reasoning and the numbers).
+		params["swap"] = strconv.Itoa(spec.MemoryMB)
 	}
 	for i, m := range spec.Mounts {
 		// backup=1 for data-bearing mounts: extra LXC mountpoints default to backup=0 = EXCLUDED

internal/reconcile/bringup_swap_test.go

@@ -0,0 +1,19 @@
+package reconcile
+
+import (
+	"testing"
+
+	"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
+)
+
+// R-1192: a new guest is created with swap = its RAM (left unset, Proxmox gives an LXC 512 MB — the measured freeze,
+// felhom.eu audits/fixes-2026-10-21/P4-freeze/). COMPANION RED-PROOF: drop params["swap"] in buildBringUpConfig.
+func TestR1192_BringUpSetsSwapToTheRAM(t *testing.T) {
+	p := buildBringUpConfig(BringUpSpec{MemoryMB: 4096}, proxmox.GuestConfig{})
+	if p["memory"] != "4096" || p["swap"] != "4096" {
+		t.Fatalf("bring-up params memory=%q swap=%q, want 4096/4096", p["memory"], p["swap"])
+	}
+	if q := buildBringUpConfig(BringUpSpec{}, proxmox.GuestConfig{}); q["swap"] != "" {
+		t.Fatalf("no memory in the spec → no swap written either, got %q", q["swap"])
+	}
+}