R-528: GET /guest/oom-kills — the kernel's per-container OOM counter of the token's guest, read on the host (no sudo)

Tests TestR528_* (fake cgroup tree), red-proved by mutation. Commit only; the agent release needs the vouch.

CHANGELOG.md

@@ -1,3 +1,16 @@
+## Unreleased (2026-10-28 night) — R-528: GET /guest/oom-kills, the kernel's OOM counter of each of the guest's containers, read on the host — commit only; the next agent release (needs the operator's vouch)
+
+- **Why:** inside a Felhom LXC guest Docker can leave `State.OOMKilled` false and send no `oom` event after a
+  memory-limit kill (measured 2026-09-15 on 9202 and 2026-09-16 on a fresh install), so the controller's OOM line
+  could not fire. The kernel counts every kill per container cgroup, and the host can read it.
+- **`internal/localapi/guest_oom.go`:** `GET /guest/oom-kills` (withGuest — the token's guest only; a cross-guest
+  `?vmid=` is 403) reads `/sys/fs/cgroup/lxc/<vmid>/ns/system.slice/docker-<64-hex>.scope/memory.events` (and
+  `ns/docker/<id>/` for the cgroupfs driver), line `oom_kill N`, and answers `{vmid, available, containers: {id: n}}`.
+  A guest with no cgroup folder → `available: false`. Read-only, no sudo: the files are 0444 and the non-root
+  felhom-agent user reads them (measured on Tester 1's host 2026-10-28). A container's scope is new at each start, so the
+  count is per container run. Tests `TestR528_*` (a fake cgroup tree; red-proved: a glob over every guest leaks another
+  guest's container; parsing `oom` instead of `oom_kill` reads the wrong number).
+
 ## Unreleased (2026-10-28 night) — R-1254 (the ceremony half, review round 2 HUNT-3 H2): the agent asks the hub which package it holds before it words a failed upload — commit only; the next agent release (needs the operator's vouch)
 
 - **The defect:** the claim's (R-978) and the confirm's (R-1151) package upload can lose its answer, or answer late

internal/localapi/guest_oom.go

@@ -0,0 +1,102 @@
+package localapi
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+)
+
+// R-528 — the OOM signal from the host side.
+//
+// MEASURED 2026-09-15 (scratch 9202, Docker 29.8.0) and 2026-09-16 (a fresh install in a nested VM): an app killed by
+// its memory limit inside a Felhom LXC guest left Docker's State.OOMKilled false and sent no `oom` event, so the
+// controller's OOM line (R-514) could not fire. The kernel's own counter does count every kill, and the host can read
+// it: /sys/fs/cgroup/lxc/<vmid>/ns/system.slice/docker-<64-hex id>.scope/memory.events, line `oom_kill N` (the
+// systemd cgroup driver; `ns/docker/<id>/` under the cgroupfs driver). Read on Tester 1's host 2026-10-28: the files
+// are mode 0444, readable by the non-root felhom-agent user — no sudo, no new root path.
+//
+// A container's scope is new at each start, so the counter is per container RUN: a count above 0 means this run had a
+// kill. The controller asks every 30 s and raises its existing app_oom line (once per run) for any such container.
+
+// readGuestOOMKills returns container id → oom_kill for one guest. available=false (no error) when the guest has no
+// cgroup folder (a VM guest, or a container that is not running).
+func readGuestOOMKills(root string, vmid int) (map[string]int64, bool, error) {
+	if root == "" {
+		root = "/sys/fs/cgroup"
+	}
+	base := filepath.Join(root, "lxc", strconv.Itoa(vmid), "ns")
+	if fi, err := os.Stat(base); err != nil || !fi.IsDir() {
+		return nil, false, nil
+	}
+	out := map[string]int64{}
+	for _, pattern := range []string{
+		filepath.Join(base, "system.slice", "docker-*.scope", "memory.events"),
+		filepath.Join(base, "docker", "*", "memory.events"),
+	} {
+		files, err := filepath.Glob(pattern)
+		if err != nil {
+			return nil, true, err
+		}
+		for _, f := range files {
+			id := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(filepath.Dir(f)), "docker-"), ".scope")
+			if !isContainerID(id) {
+				continue
+			}
+			n, err := readOOMKill(f)
+			if err != nil {
+				continue // a container that stopped between the glob and the read
+			}
+			out[id] = n
+		}
+	}
+	return out, true, nil
+}
+
+func isContainerID(s string) bool {
+	if len(s) != 64 {
+		return false
+	}
+	for _, c := range s {
+		if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') {
+			return false
+		}
+	}
+	return true
+}
+
+func readOOMKill(path string) (int64, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		k, v, ok := strings.Cut(strings.TrimSpace(sc.Text()), " ")
+		if ok && k == "oom_kill" {
+			return strconv.ParseInt(strings.TrimSpace(v), 10, 64)
+		}
+	}
+	if err := sc.Err(); err != nil {
+		return 0, err
+	}
+	return 0, fmt.Errorf("no oom_kill line in %s", path)
+}
+
+// handleGuestOOMKills is GET /guest/oom-kills: {"vmid", "available", "containers": {id: oom_kill}}.
+func (s *Server) handleGuestOOMKills(w http.ResponseWriter, r *http.Request, vmid int) {
+	kills, avail, err := readGuestOOMKills(s.cgroupRoot, vmid)
+	if err != nil {
+		s.logger.Warn("local-api: guest oom-kills read failed (R-528)", "vmid", vmid, "err", err)
+		writeErr(w, http.StatusInternalServerError, "the guest's OOM counters could not be read")
+		return
+	}
+	if kills == nil {
+		kills = map[string]int64{}
+	}
+	writeOK(w, map[string]any{"vmid": vmid, "available": avail, "containers": kills})
+}

internal/localapi/guest_oom_test.go

@@ -0,0 +1,101 @@
+package localapi
+
+import (
+	"encoding/json"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+// R-528: Docker does not always report an OOM kill inside a Felhom LXC guest (OOMKilled stays false, no `oom` event —
+// measured 2026-09-15/16 on two boxes), but the kernel's own counter does count it, and it is readable from the host:
+// /sys/fs/cgroup/lxc/<vmid>/ns/system.slice/docker-<id>.scope/memory.events, line `oom_kill N`. GET /guest/oom-kills
+// serves the asking guest's counters, and only that guest's.
+
+func fakeCgroup(t *testing.T, root string, vmid, id, events string) {
+	t.Helper()
+	dir := filepath.Join(root, "lxc", vmid, "ns", "system.slice", "docker-"+id+".scope")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "memory.events"), []byte(events), 0o444); err != nil {
+		t.Fatal(err)
+	}
+}
+
+const idA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+const idB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+const idOther = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"
+
+// RED-PROOF: serve the counters of every guest under lxc/ (drop the vmid from the glob) → the other guest's container
+// appears → FAILS; parse `oom ` instead of `oom_kill` → idA reads 9 → FAILS.
+func TestR528_GuestOOMKillsServesThisGuestsCountersOnly(t *testing.T) {
+	root := t.TempDir()
+	fakeCgroup(t, root, "100", idA, "low 0\nhigh 0\nmax 4\noom 9\noom_kill 3\noom_group_kill 0\n")
+	fakeCgroup(t, root, "100", idB, "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\n")
+	fakeCgroup(t, root, "200", idOther, "oom_kill 7\n")
+	// a scope that is not a container (no 64-hex id) and a cgroupfs-driver layout container
+	if err := os.MkdirAll(filepath.Join(root, "lxc", "100", "ns", "system.slice", "cron.service"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	dfDir := filepath.Join(root, "lxc", "100", "ns", "docker", idOther[:8]+strings.Repeat("d", 56))
+	if err := os.MkdirAll(dfDir, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	_ = os.WriteFile(filepath.Join(dfDir, "memory.events"), []byte("oom_kill 2\n"), 0o444)
+
+	got, avail, err := readGuestOOMKills(root, 100)
+	if err != nil || !avail {
+		t.Fatalf("read: avail=%v err=%v", avail, err)
+	}
+	want := map[string]int64{idA: 3, idB: 0, idOther[:8] + strings.Repeat("d", 56): 2}
+	if len(got) != len(want) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+	for k, v := range want {
+		if got[k] != v {
+			t.Fatalf("got %v, want %v", got, want)
+		}
+	}
+	if _, leaked := got[idOther]; leaked {
+		t.Fatal("another guest's container leaked into this guest's answer")
+	}
+	// A guest with no cgroup folder (a VM, or not running): not available, no error.
+	if _, avail, err := readGuestOOMKills(root, 300); avail || err != nil {
+		t.Fatalf("a guest without a cgroup folder: avail=%v err=%v", avail, err)
+	}
+}
+
+// Through the route, with the guest's own token.
+func TestR528_TheRouteAnswersTheTokensGuest(t *testing.T) {
+	root := t.TempDir()
+	fakeCgroup(t, root, "8200", idA, "oom_kill 5\n")
+	fakeCgroup(t, root, "9300", idB, "oom_kill 1\n") // token B's guest: never in A's answer
+	srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil)
+	srv.cgroupRoot = root
+	w := do(t, srv.Handler(), "GET", "/guest/oom-kills", "A", "")
+	if w.Code != http.StatusOK {
+		t.Fatalf("route: %d %s", w.Code, w.Body.String())
+	}
+	var env struct {
+		Data struct {
+			VMID       int              `json:"vmid"`
+			Available  bool             `json:"available"`
+			Containers map[string]int64 `json:"containers"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil {
+		t.Fatal(err)
+	}
+	if env.Data.VMID != 8200 || !env.Data.Available || len(env.Data.Containers) != 1 || env.Data.Containers[idA] != 5 {
+		t.Fatalf("answer %s — want guest 8200's one container with 5", w.Body.String())
+	}
+	if w := do(t, srv.Handler(), "GET", "/guest/oom-kills?vmid=9300", "A", ""); w.Code != http.StatusForbidden {
+		t.Fatalf("a cross-guest ask: %d, want 403", w.Code)
+	}
+	if w := do(t, srv.Handler(), "GET", "/guest/oom-kills", "", ""); w.Code != http.StatusUnauthorized {
+		t.Fatalf("no token: %d", w.Code)
+	}
+}

internal/localapi/server.go

@@ -459,6 +459,8 @@ type Server struct {
 	escrowLookPath func(file string) (string, error)
 	// statFile reports whether a path exists (preflight's staged-secret item; tests inject).
 	statFile func(path string) bool
+	// cgroupRoot is where the host's cgroup tree is mounted (R-528, GET /guest/oom-kills); "" = /sys/fs/cgroup. Tests set it.
+	cgroupRoot string
 
 	baseCtx context.Context // for fire-and-forget backups; set in Run
 }
@@ -576,6 +578,9 @@ func (s *Server) Handler() http.Handler {
 	mux.HandleFunc("GET /host/crash-guard", s.withGuest(s.handleCrashGuard))
 	// R-1247: the weekly upload line test's last measurement — the controller's qBittorrent limit and upload watch
 	// read it. Read-only; never measured (or the job not wired) = present:false.
+	// R-528: the kernel's OOM-kill counter of each of the guest's containers, read on the host (Docker's own flag and
+	// event can stay silent inside an LXC guest). Read-only; the asking guest's containers only.
+	mux.HandleFunc("GET /guest/oom-kills", s.withGuest(s.handleGuestOOMKills))
 	mux.HandleFunc("GET /host/line-test", s.withGuest(s.handleLineTest))
 	mux.HandleFunc("POST /host/line-test/run", s.withGuest(s.handleLineTestRun)) // R-1250: a test on request
 	// Disk management (slice 8C) — self-scoped; format routes through the data-bearing classifier+gate.