diff --git a/CHANGELOG.md b/CHANGELOG.md index e8f7e60..013deea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,23 @@ to 4096 MB live ended it within a minute while the uploads ran on. new swap assertion — each red-proved (the write removed → `raised 0, want 2`; `swap=""` on both other paths). - Privilege: `swap` is under `VM.Config.Memory`, which the RAM resize already uses; no role, sudoers or bundle change. +Added the night of 2026-10-21 (still unreleased, the same release): + +- **The host's swap is reported and checked (R-1192 follow-up).** The guest fix moves a memory burst onto the HOST's + swap; a host with none gets nothing from it. The host report carries `host.host_swap_mb` (Proxmox node status — the + kernel's SwapTotal, the same number as `/proc/meminfo`) and `host.guest_swap_mb` (the sum of the guests' `swap`; each + guest spec also carries `swap_bytes`). When the first is below the second the agent logs one WARN (and one INFO when + it is covered again); the hub raises `host_swap_short`. `GET /guest/memory` carries `host_swap_mb` and + `guest_swap_mb` for the controller's install page. The agent never creates host swap (a decision — felhom.eu + register). Measured the same night: our installer gives an 8 GB / 256 GB machine 7.75 GB of host swap. +- **R-1194 — no false „operator access degraded" on a fresh box.** The first host report was sent before the + felhom-sshd loop's first pass, read a config that was never rendered („config invalid, port 0") and the hub mailed + `oob_degraded` — reproduced on a fresh install 2026-10-21 (the report at 20:34:00, the apply 0.4 s later, the daemon + healthy; the check runs `sshd -t` through sudo, so the non-root theory was false). The loop now reports no OOB stanza + until its first pass has run (a failed pass counts, so a real lasting failure still reaches the hub). +- Tests `TestR1192_HostReportCarriesSwapAndWarnsOnce`, `TestR1192_GuestMemoryCarriesHostAndGuestSwap`, + `TestR1194_NoOOBStanzaBeforeTheFirstPass` — each red-proved. + ## v0.161.0 — three fixes (R-921, R-1070, R-1010); two texts made true after 0.160.0 (2026-10-20) — RELEASED on the operator's yes, NOT VOUCHED Released 2026-10-20 (operator yes, the morning answers) with `scripts/release-agent.sh` from a clean clone at `5152123`: diff --git a/internal/felhomsshd/loop.go b/internal/felhomsshd/loop.go index 526314c..c05199a 100644 --- a/internal/felhomsshd/loop.go +++ b/internal/felhomsshd/loop.go @@ -22,6 +22,11 @@ type Loop struct { mu sync.Mutex fetched bool // a desired-state document has been delivered (false = never a teardown signal) block *hub.WireWireguard + // passed (R-1194) is set once the first reconcile pass has run. Until then the instance has never been rendered, + // so its health says nothing — and the first host report, sent before that pass, used to read „config invalid, + // port 0" and raise a false oob_degraded on EVERY fresh box (rehearsal 2026-10-20; reproduced 2026-10-21: the + // report at 20:34:00 read an absent config, the apply landed 0.4 s later). + passed bool nudge chan struct{} } @@ -85,6 +90,7 @@ func (l *Loop) reconcile(ctx context.Context) { if !fetched { applyBlock = nil } + defer l.markPassed() port, err := l.mgr.Apply(ctx, applyBlock) if err != nil { return // Apply logged; a claim/exhaustion or install error — retry next tick @@ -106,8 +112,23 @@ func oobPeerIP(block *hub.WireWireguard) string { return block.OOBPeerIP } -// OOBStatus implements the hub collector's reporter seam (Part 4). +// markPassed records that a reconcile pass has run (R-1194) — after a failed Apply too, so a real, lasting failure +// is still reported from the next report on. +func (l *Loop) markPassed() { + l.mu.Lock() + l.passed = true + l.mu.Unlock() +} + +// OOBStatus implements the hub collector's reporter seam (Part 4). nil — no stanza, which the hub does not evaluate — +// until the first reconcile pass has run (R-1194). func (l *Loop) OOBStatus(ctx context.Context) *hub.OOBStatus { + l.mu.Lock() + passed := l.passed + l.mu.Unlock() + if !passed { + return nil + } _, block := l.snapshot() return l.mgr.Status(ctx, block) } diff --git a/internal/felhomsshd/r1194_first_report_test.go b/internal/felhomsshd/r1194_first_report_test.go new file mode 100644 index 0000000..37468f7 --- /dev/null +++ b/internal/felhomsshd/r1194_first_report_test.go @@ -0,0 +1,29 @@ +package felhomsshd + +import ( + "context" + "testing" + "time" +) + +// R-1194: on every fresh box the first host report — sent before the OOB loop's first pass — read „config invalid, +// port 0" (the config was not rendered yet) and the hub mailed oob_degraded. Reproduced 2026-10-21 on a fresh box +// (felhom.eu audits/night-2026-10-21/P5-checks/): the report at 20:34:00, the apply 0.4 s later, the daemon healthy. +// The loop now reports NO stanza (the hub does not evaluate a missing one) until a pass has run — a failed pass +// counts, so a real lasting failure is still reported from the next report on. +// RED-PROOF: with OOBStatus not gated on `passed`, the first assertion fails (a stanza with config_invalid). +func TestR1194_NoOOBStanzaBeforeTheFirstPass(t *testing.T) { + r := newScriptRunner() + m := NewManager(r, t.TempDir(), nil) + m.isActive = func(context.Context) bool { return false } + m.isFailed = func(context.Context) bool { return false } + m.now = func() time.Time { return time.Unix(1783270000, 0) } + l := NewLoop(m, nil, time.Minute, nil) + if st := l.OOBStatus(context.Background()); st != nil { + t.Fatalf("a stanza before the first pass reads a config that was never rendered: %+v", st) + } + l.reconcile(context.Background()) // fails or succeeds — either way a pass has run + if st := l.OOBStatus(context.Background()); st == nil { + t.Fatal("after the first pass the health must be reported (a real failure must reach the hub)") + } +} diff --git a/internal/hub/collect.go b/internal/hub/collect.go index 4cf836c..4a6ee69 100644 --- a/internal/hub/collect.go +++ b/internal/hub/collect.go @@ -11,6 +11,7 @@ import ( "os" "strings" "sync" + "sync/atomic" "time" "gitea.dooplex.hu/admin/felhom-agent/internal/capability" @@ -136,6 +137,8 @@ type Collector struct { selfSHA func() string // R-349: sha256 of the running binary; default runningBinarySHA256 logger *slog.Logger now func() time.Time + // swapShortWarned (R-1192): the host-swap-short WARN is logged once per state change, not every report. + swapShortWarned atomic.Bool } // NewCollector builds a collector. hostID echoes config.Hub.HostID; agentVersion is @@ -375,6 +378,8 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { } host := hostMetrics(c.px.Node(), ns) + guests := c.collectGuests(ctx) + c.applySwap(&host, ns, guests) // R-1192: host swap vs the guests' swap allowance host.CPUTempC = c.cpuTempC(ctx) // slice 9: operator freebie — temp now rides the hub report too host.WrapperSHA256 = pbsWrapperSHA256() // R-50b(a): make privileged-artifact drift answerable report := &HostReport{ @@ -383,7 +388,7 @@ func (c *Collector) Collect(ctx context.Context) (*HostReport, error) { AgentVersion: c.agentVersion, AgentSHA256: c.agentSHA256(), Host: host, - Guests: c.collectGuests(ctx), + Guests: guests, // storage_targets populated this slice (slice 5) via the observer; the rest stay // defined-but-empty (slice 6). Non-nil so they marshal as []. StorageTargets: c.collectStorage(ctx), @@ -567,6 +572,7 @@ func (c *Collector) collectGuests(ctx context.Context) []Guest { Cores: cfg.Cores, MemoryBytes: g.MaxMem, DiskBytes: g.MaxDisk, + SwapBytes: cfg.Swap * (1 << 20), } } guests = append(guests, entry) diff --git a/internal/hub/hostswap.go b/internal/hub/hostswap.go new file mode 100644 index 0000000..91010af --- /dev/null +++ b/internal/hub/hostswap.go @@ -0,0 +1,39 @@ +package hub + +import ( + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// R-1192 follow-up (night 2026-10-21): the guest swap fix (every guest may swap as much as its RAM, +// localapi/guestswap.go) moves a memory burst onto the HOST's swap. Measured on demo-hp the same day: our installer +// gives an 8 GB / 256 GB machine 7.75 GB of host swap, so a 4096 MB guest's 4096 MB swap fits. A host with no swap +// (a BYO install, or swap turned off) gets nothing from the fix. So the report carries both numbers, the agent says it +// once in its log, and the hub raises host_swap_short. The agent never creates host swap itself: changing a host its +// owner set up is a decision (felhom.eu register). + +// HostSwapShort reports whether the host's swap is below the guests' swap allowance. A host whose guests have no +// swap allowance (0) is never short. +func HostSwapShort(hostMB, guestMB int64) bool { return guestMB > 0 && hostMB < guestMB } + +// applySwap fills the host block's swap numbers from the node status and the guests' specs, and logs the short +// state once per change. +func (c *Collector) applySwap(h *HostMetrics, ns proxmox.NodeStatus, guests []Guest) { + hostMB := ns.Swap.Total / (1 << 20) + var guestMB int64 + for _, g := range guests { + if g.Spec != nil { + guestMB += g.Spec.SwapBytes / (1 << 20) + } + } + h.HostSwapMB, h.GuestSwapMB = &hostMB, &guestMB + if HostSwapShort(hostMB, guestMB) { + if c.swapShortWarned.CompareAndSwap(false, true) { + c.logger.Warn("host-swap: the host's swap is below its guests' swap allowance — a memory burst can freeze the box (R-1192); the hub is told (host_swap_short)", + "host_swap_mb", hostMB, "guest_swap_mb", guestMB) + } + return + } + if c.swapShortWarned.CompareAndSwap(true, false) { + c.logger.Info("host-swap: the host's swap covers its guests' swap allowance again", "host_swap_mb", hostMB, "guest_swap_mb", guestMB) + } +} diff --git a/internal/hub/r1192_hostswap_test.go b/internal/hub/r1192_hostswap_test.go new file mode 100644 index 0000000..82223d5 --- /dev/null +++ b/internal/hub/r1192_hostswap_test.go @@ -0,0 +1,42 @@ +package hub + +import ( + "bytes" + "log/slog" + "strings" + "testing" + + "gitea.dooplex.hu/admin/felhom-agent/internal/proxmox" +) + +// R-1192 follow-up (night 2026-10-21): the host report carries the host's swap and the guests' swap allowance, and the +// agent says ONCE in its log when the host's swap is below it (and once when it is covered again). +// RED-PROOF: with applySwap not called the numbers are nil; with the CompareAndSwap removed the WARN repeats. +func TestR1192_HostReportCarriesSwapAndWarnsOnce(t *testing.T) { + var buf bytes.Buffer + c := &Collector{logger: slog.New(slog.NewTextHandler(&buf, nil))} + guests := []Guest{{VMID: 9201, Spec: &GuestSpec{SwapBytes: 4096 << 20}}, {VMID: 9202, Spec: nil}} + var ns proxmox.NodeStatus // a host with no swap + for i := 0; i < 3; i++ { + h := HostMetrics{} + c.applySwap(&h, ns, guests) + if h.HostSwapMB == nil || h.GuestSwapMB == nil || *h.HostSwapMB != 0 || *h.GuestSwapMB != 4096 { + t.Fatalf("report numbers: host %v guest %v", h.HostSwapMB, h.GuestSwapMB) + } + } + if n := strings.Count(buf.String(), "host-swap: the host's swap is below"); n != 1 { + t.Fatalf("the short WARN must be logged once, got %d:\n%s", n, buf.String()) + } + ns.Swap.Total = 7935 << 20 // the installer's 8 GB host + h := HostMetrics{} + c.applySwap(&h, ns, guests) + if *h.HostSwapMB != 7935 || HostSwapShort(*h.HostSwapMB, *h.GuestSwapMB) { + t.Fatalf("a covered host must not be short: %v / %v", *h.HostSwapMB, *h.GuestSwapMB) + } + if !strings.Contains(buf.String(), "covers its guests' swap allowance again") { + t.Fatalf("the recovery must be said once:\n%s", buf.String()) + } + if HostSwapShort(0, 0) { + t.Fatal("a host whose guests have no swap allowance is never short") + } +} diff --git a/internal/hub/report.go b/internal/hub/report.go index 8a9258f..8e6f261 100644 --- a/internal/hub/report.go +++ b/internal/hub/report.go @@ -338,6 +338,11 @@ type HostMetrics struct { // // Empty = unreadable/absent, which the hub treats as UNKNOWN, never as drift. WrapperSHA256 string `json:"wrapper_sha256,omitempty"` + // HostSwapMB / GuestSwapMB (R-1192 follow-up, night 2026-10-21): the host's swap (Proxmox node status — the + // kernel's SwapTotal) and the sum of its guests' swap allowance. The guest swap fix (swap = RAM) needs swap ON THE + // HOST behind it; the hub raises host_swap_short when the first is below the second. nil = not read (older agent). + HostSwapMB *int64 `json:"host_swap_mb,omitempty"` + GuestSwapMB *int64 `json:"guest_swap_mb,omitempty"` } // Guest is one LXC. The agent reports vmid; the hub derives the guest PK @@ -355,6 +360,7 @@ type GuestSpec struct { Cores int `json:"cores"` MemoryBytes int64 `json:"memory_bytes"` DiskBytes int64 `json:"disk_bytes"` + SwapBytes int64 `json:"swap_bytes,omitempty"` // R-1192: the guest's swap allowance (PVE config `swap`, MB → bytes) } // Cloudflared is the box's tunnel (R-841, agent v0.141.0): the cloudflared container in the customer guest. diff --git a/internal/localapi/guestmemory.go b/internal/localapi/guestmemory.go index 96fdec6..d61764a 100644 --- a/internal/localapi/guestmemory.go +++ b/internal/localapi/guestmemory.go @@ -57,6 +57,10 @@ type MemoryInfo struct { MaxMB int64 `json:"max_mb"` FloorMB int64 `json:"floor_mb"` Running bool `json:"running"` + // HostSwapMB / GuestSwapMB (R-1192 follow-up): the host's swap and this guest's swap allowance, so the controller + // can tell the household when the host has too little swap behind the guest (its install page). nil = not read. + HostSwapMB *int64 `json:"host_swap_mb,omitempty"` + GuestSwapMB *int64 `json:"guest_swap_mb,omitempty"` } // memoryResizeRequest is the POST /guest/memory body. @@ -94,6 +98,8 @@ type memoryBounds struct { maxMB int64 floorMB int64 running bool + hostSwapMB int64 + guestSwapMB int64 } func (b memoryBounds) refusal(code string) memoryRefusal { @@ -104,7 +110,9 @@ func (b memoryBounds) refusal(code string) memoryRefusal { } func (b memoryBounds) info(vmid int) MemoryInfo { + hostSwap, guestSwap := b.hostSwapMB, b.guestSwapMB return MemoryInfo{ + HostSwapMB: &hostSwap, GuestSwapMB: &guestSwap, VMID: vmid, AllocatedMB: b.allocatedMB, UsageMB: b.usageMB, HostTotalMB: b.hostTotalMB, MinMB: b.minMB, MaxMB: b.maxMB, FloorMB: b.floorMB, Running: b.running, } @@ -139,6 +147,8 @@ func (s *Server) readMemoryBounds(ctx context.Context, vmid int) (memoryBounds, hostTotalMB: node.Memory.Total / mib, // bytes → MB, floor (conservative for max) minMB: minGuestMemoryMB, running: st.Status == "running", + hostSwapMB: node.Swap.Total / mib, // R-1192 follow-up: the kernel's SwapTotal, as Proxmox reports it + guestSwapMB: cfg.Swap, // PVE config swap is MB } b.maxMB = b.hostTotalMB - hostReserveMB b.floorMB = b.usageMB + shrinkUsageMarginMB diff --git a/internal/localapi/guestmemory_test.go b/internal/localapi/guestmemory_test.go index a956807..78d9790 100644 --- a/internal/localapi/guestmemory_test.go +++ b/internal/localapi/guestmemory_test.go @@ -25,12 +25,14 @@ type fakeMemory struct { setUPID string setErr error applyReflect bool + hostSwapB int64 // R-1192 follow-up: node swap total + guestSwapMB int64 } func (f *fakeMemory) GuestConfig(_ context.Context, _ int) (proxmox.GuestConfig, error) { f.mu.Lock() defer f.mu.Unlock() - return proxmox.GuestConfig{Memory: f.allocMB}, nil + return proxmox.GuestConfig{Memory: f.allocMB, Swap: f.guestSwapMB}, nil } func (f *fakeMemory) GuestStatus(_ context.Context, vmid int) (proxmox.Guest, error) { f.mu.Lock() @@ -42,6 +44,7 @@ func (f *fakeMemory) NodeStatus(_ context.Context) (proxmox.NodeStatus, error) { defer f.mu.Unlock() var ns proxmox.NodeStatus ns.Memory.Total = f.hostTotalB + ns.Swap.Total = f.hostSwapB return ns, nil } func (f *fakeMemory) SetConfig(_ context.Context, _ int, params map[string]string) (string, error) { diff --git a/internal/localapi/r1192_hostswap_test.go b/internal/localapi/r1192_hostswap_test.go new file mode 100644 index 0000000..5db2767 --- /dev/null +++ b/internal/localapi/r1192_hostswap_test.go @@ -0,0 +1,39 @@ +package localapi + +import ( + "encoding/json" + "testing" +) + +// R-1192 follow-up (night 2026-10-21): GET /guest/memory carries the host's swap and this guest's swap allowance, so +// the controller's install page can say when the host has no swap behind the guest. 0 is a real answer (no swap); +// absent means an older agent. +// RED-PROOF: without the two MemoryInfo fields the keys are missing → fails on „host_swap_mb absent". +func TestR1192_GuestMemoryCarriesHostAndGuestSwap(t *testing.T) { + for _, c := range []struct { + name string + hostB, guestMB int64 + wantHost, wantG int64 + }{ + {"installer host (7.75 GB swap), guest swap = RAM", 7935 * mib, 8192, 7935, 8192}, + {"a host with NO swap", 0, 8192, 0, 8192}, + } { + f := stdFixture() + f.hostSwapB, f.guestSwapMB = c.hostB, c.guestMB + w := do(t, memServer(t, f).Handler(), "GET", "/guest/memory", "A", "") + var m struct { + Data map[string]any `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &m); err != nil { + t.Fatal(err) + } + hs, ok := m.Data["host_swap_mb"].(float64) + if !ok { + t.Fatalf("%s: host_swap_mb absent: %s", c.name, w.Body.String()) + } + gs, _ := m.Data["guest_swap_mb"].(float64) + if int64(hs) != c.wantHost || int64(gs) != c.wantG { + t.Fatalf("%s: host %v guest %v, want %d / %d", c.name, hs, gs, c.wantHost, c.wantG) + } + } +}