74 lines
3.5 KiB
Go
74 lines
3.5 KiB
Go
package localapi
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"log/slog"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||
)
|
||
|
||
// ── R-1192 (2026-10-21): the guest may swap as much as its RAM ────────────────────────────────────────────────────
|
||
//
|
||
// MEASURED (felhom.eu audits/fixes-2026-10-21/P4-freeze/): on the minimum machine (8 GB host, guest 4096 MB) the
|
||
// household's first uploads made Immich load its analysis models (+1.15 GB) on top of 8 apps' 1.75 GB; the guest hit
|
||
// its 4096 MB limit with its swap capped at Proxmox's LXC default of 512 MB — and full. The kernel could then evict
|
||
// only file pages: ~360 MB was left for the program code of ~25 containers, re-read from disk without end (150 000–
|
||
// 600 000 refaults and ~1 GB of reads per 10 s; memory „full" pressure 92–95 %, load 155, the dashboard dead, 0 OOM
|
||
// kills). Raising the cap to 4096 MB live ended it in under a minute (memory „full" 96 % → 1.7 %, dashboard back
|
||
// 42 s later) while the uploads ran on. The host had 7.4 GB of swap free the whole time.
|
||
//
|
||
// The agent never set `swap` at all, so every guest had the 512 MB default. Now:
|
||
// - a new guest is created with swap = its memory (reconcile.bringup);
|
||
// - a RAM resize sets swap with the memory (handleGuestMemoryResize);
|
||
// - at agent start, every LXC in the felhom pool whose swap is below its memory is raised to it (this file).
|
||
// Raising swap is a live cgroup change (memory.swap.max), no restart. It never lowers a swap someone set higher.
|
||
|
||
// GuestSwapMB is the swap a guest of memoryMB gets: the same as its RAM.
|
||
func GuestSwapMB(memoryMB int64) int64 { return memoryMB }
|
||
|
||
// SwapOps is the Proxmox surface EnsureGuestSwap needs (satisfied by *proxmox.Client).
|
||
type SwapOps interface {
|
||
Pool(ctx context.Context, name string) (proxmox.PoolInfo, error)
|
||
GuestConfig(ctx context.Context, vmid int) (proxmox.GuestConfig, error)
|
||
SetConfig(ctx context.Context, vmid int, params map[string]string) (string, error)
|
||
WaitTask(ctx context.Context, upid string, opts proxmox.WaitOptions) (proxmox.TaskStatus, error)
|
||
}
|
||
|
||
// EnsureGuestSwap raises the swap of every LXC guest in pool to GuestSwapMB(memory) when it is below. Returns how
|
||
// many guests were raised. One guest's failure is logged and the others are still handled.
|
||
func EnsureGuestSwap(ctx context.Context, ops SwapOps, pool string, logger *slog.Logger) (int, error) {
|
||
p, err := ops.Pool(ctx, pool)
|
||
if err != nil {
|
||
return 0, fmt.Errorf("guest swap: read pool %s: %w", pool, err)
|
||
}
|
||
raised := 0
|
||
for _, m := range p.Members {
|
||
if m.Type != "lxc" || m.VMID <= 0 {
|
||
continue
|
||
}
|
||
cfg, err := ops.GuestConfig(ctx, m.VMID)
|
||
if err != nil {
|
||
logger.Warn("guest-swap: config read failed (left as it is)", "vmid", m.VMID, "err", err)
|
||
continue
|
||
}
|
||
want := GuestSwapMB(cfg.Memory)
|
||
if cfg.Memory <= 0 || cfg.Swap >= want {
|
||
logger.Debug("guest-swap: nothing to do", "vmid", m.VMID, "memory_mb", cfg.Memory, "swap_mb", cfg.Swap)
|
||
continue
|
||
}
|
||
upid, err := ops.SetConfig(ctx, m.VMID, map[string]string{"swap": fmt.Sprintf("%d", want)})
|
||
if err == nil && upid != "" {
|
||
_, err = ops.WaitTask(ctx, upid, proxmox.WaitOptions{})
|
||
}
|
||
if err != nil {
|
||
logger.Warn("guest-swap: raise failed (left as it is)", "vmid", m.VMID, "from_mb", cfg.Swap, "to_mb", want, "err", err)
|
||
continue
|
||
}
|
||
raised++
|
||
logger.Info("guest-swap: raised (R-1192 — a guest swap cap below its RAM turned a memory burst into a freeze)",
|
||
"vmid", m.VMID, "from_mb", cfg.Swap, "to_mb", want)
|
||
}
|
||
return raised, nil
|
||
}
|