From 6f933d6ee3c0b83ae22a9ca58a504996efc882e2 Mon Sep 17 00:00:00 2001 From: kisfenyo Date: Wed, 28 Oct 2026 23:22:40 +0100 Subject: [PATCH] =?UTF-8?q?R-528:=20GET=20/guest/oom-kills=20=E2=80=94=20t?= =?UTF-8?q?he=20kernel's=20per-container=20OOM=20counter=20of=20the=20toke?= =?UTF-8?q?n's=20guest,=20read=20on=20the=20host=20(no=20sudo)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tests TestR528_* (fake cgroup tree), red-proved by mutation. Commit only; the agent release needs the vouch. --- CHANGELOG.md | 13 ++++ internal/localapi/guest_oom.go | 102 ++++++++++++++++++++++++++++ internal/localapi/guest_oom_test.go | 101 +++++++++++++++++++++++++++ internal/localapi/server.go | 5 ++ 4 files changed, 221 insertions(+) create mode 100644 internal/localapi/guest_oom.go create mode 100644 internal/localapi/guest_oom_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fa507ec..7bdfc72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,16 @@ +## Unreleased (2026-10-28 night) — R-528: GET /guest/oom-kills, the kernel's OOM counter of each of the guest's containers, read on the host — commit only; the next agent release (needs the operator's vouch) + +- **Why:** inside a Felhom LXC guest Docker can leave `State.OOMKilled` false and send no `oom` event after a + memory-limit kill (measured 2026-09-15 on 9202 and 2026-09-16 on a fresh install), so the controller's OOM line + could not fire. The kernel counts every kill per container cgroup, and the host can read it. +- **`internal/localapi/guest_oom.go`:** `GET /guest/oom-kills` (withGuest — the token's guest only; a cross-guest + `?vmid=` is 403) reads `/sys/fs/cgroup/lxc//ns/system.slice/docker-<64-hex>.scope/memory.events` (and + `ns/docker//` for the cgroupfs driver), line `oom_kill N`, and answers `{vmid, available, containers: {id: n}}`. + A guest with no cgroup folder → `available: false`. Read-only, no sudo: the files are 0444 and the non-root + felhom-agent user reads them (measured on Tester 1's host 2026-10-28). A container's scope is new at each start, so the + count is per container run. Tests `TestR528_*` (a fake cgroup tree; red-proved: a glob over every guest leaks another + guest's container; parsing `oom` instead of `oom_kill` reads the wrong number). + ## Unreleased (2026-10-28 night) — R-1254 (the ceremony half, review round 2 HUNT-3 H2): the agent asks the hub which package it holds before it words a failed upload — commit only; the next agent release (needs the operator's vouch) - **The defect:** the claim's (R-978) and the confirm's (R-1151) package upload can lose its answer, or answer late diff --git a/internal/localapi/guest_oom.go b/internal/localapi/guest_oom.go new file mode 100644 index 0000000..7c77227 --- /dev/null +++ b/internal/localapi/guest_oom.go @@ -0,0 +1,102 @@ +package localapi + +import ( + "bufio" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" +) + +// R-528 — the OOM signal from the host side. +// +// MEASURED 2026-09-15 (scratch 9202, Docker 29.8.0) and 2026-09-16 (a fresh install in a nested VM): an app killed by +// its memory limit inside a Felhom LXC guest left Docker's State.OOMKilled false and sent no `oom` event, so the +// controller's OOM line (R-514) could not fire. The kernel's own counter does count every kill, and the host can read +// it: /sys/fs/cgroup/lxc//ns/system.slice/docker-<64-hex id>.scope/memory.events, line `oom_kill N` (the +// systemd cgroup driver; `ns/docker//` under the cgroupfs driver). Read on Tester 1's host 2026-10-28: the files +// are mode 0444, readable by the non-root felhom-agent user — no sudo, no new root path. +// +// A container's scope is new at each start, so the counter is per container RUN: a count above 0 means this run had a +// kill. The controller asks every 30 s and raises its existing app_oom line (once per run) for any such container. + +// readGuestOOMKills returns container id → oom_kill for one guest. available=false (no error) when the guest has no +// cgroup folder (a VM guest, or a container that is not running). +func readGuestOOMKills(root string, vmid int) (map[string]int64, bool, error) { + if root == "" { + root = "/sys/fs/cgroup" + } + base := filepath.Join(root, "lxc", strconv.Itoa(vmid), "ns") + if fi, err := os.Stat(base); err != nil || !fi.IsDir() { + return nil, false, nil + } + out := map[string]int64{} + for _, pattern := range []string{ + filepath.Join(base, "system.slice", "docker-*.scope", "memory.events"), + filepath.Join(base, "docker", "*", "memory.events"), + } { + files, err := filepath.Glob(pattern) + if err != nil { + return nil, true, err + } + for _, f := range files { + id := strings.TrimSuffix(strings.TrimPrefix(filepath.Base(filepath.Dir(f)), "docker-"), ".scope") + if !isContainerID(id) { + continue + } + n, err := readOOMKill(f) + if err != nil { + continue // a container that stopped between the glob and the read + } + out[id] = n + } + } + return out, true, nil +} + +func isContainerID(s string) bool { + if len(s) != 64 { + return false + } + for _, c := range s { + if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'f') { + return false + } + } + return true +} + +func readOOMKill(path string) (int64, error) { + f, err := os.Open(path) + if err != nil { + return 0, err + } + defer f.Close() + sc := bufio.NewScanner(f) + for sc.Scan() { + k, v, ok := strings.Cut(strings.TrimSpace(sc.Text()), " ") + if ok && k == "oom_kill" { + return strconv.ParseInt(strings.TrimSpace(v), 10, 64) + } + } + if err := sc.Err(); err != nil { + return 0, err + } + return 0, fmt.Errorf("no oom_kill line in %s", path) +} + +// handleGuestOOMKills is GET /guest/oom-kills: {"vmid", "available", "containers": {id: oom_kill}}. +func (s *Server) handleGuestOOMKills(w http.ResponseWriter, r *http.Request, vmid int) { + kills, avail, err := readGuestOOMKills(s.cgroupRoot, vmid) + if err != nil { + s.logger.Warn("local-api: guest oom-kills read failed (R-528)", "vmid", vmid, "err", err) + writeErr(w, http.StatusInternalServerError, "the guest's OOM counters could not be read") + return + } + if kills == nil { + kills = map[string]int64{} + } + writeOK(w, map[string]any{"vmid": vmid, "available": avail, "containers": kills}) +} diff --git a/internal/localapi/guest_oom_test.go b/internal/localapi/guest_oom_test.go new file mode 100644 index 0000000..a808403 --- /dev/null +++ b/internal/localapi/guest_oom_test.go @@ -0,0 +1,101 @@ +package localapi + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + "testing" +) + +// R-528: Docker does not always report an OOM kill inside a Felhom LXC guest (OOMKilled stays false, no `oom` event — +// measured 2026-09-15/16 on two boxes), but the kernel's own counter does count it, and it is readable from the host: +// /sys/fs/cgroup/lxc//ns/system.slice/docker-.scope/memory.events, line `oom_kill N`. GET /guest/oom-kills +// serves the asking guest's counters, and only that guest's. + +func fakeCgroup(t *testing.T, root string, vmid, id, events string) { + t.Helper() + dir := filepath.Join(root, "lxc", vmid, "ns", "system.slice", "docker-"+id+".scope") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "memory.events"), []byte(events), 0o444); err != nil { + t.Fatal(err) + } +} + +const idA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +const idB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" +const idOther = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + +// RED-PROOF: serve the counters of every guest under lxc/ (drop the vmid from the glob) → the other guest's container +// appears → FAILS; parse `oom ` instead of `oom_kill` → idA reads 9 → FAILS. +func TestR528_GuestOOMKillsServesThisGuestsCountersOnly(t *testing.T) { + root := t.TempDir() + fakeCgroup(t, root, "100", idA, "low 0\nhigh 0\nmax 4\noom 9\noom_kill 3\noom_group_kill 0\n") + fakeCgroup(t, root, "100", idB, "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\n") + fakeCgroup(t, root, "200", idOther, "oom_kill 7\n") + // a scope that is not a container (no 64-hex id) and a cgroupfs-driver layout container + if err := os.MkdirAll(filepath.Join(root, "lxc", "100", "ns", "system.slice", "cron.service"), 0o755); err != nil { + t.Fatal(err) + } + dfDir := filepath.Join(root, "lxc", "100", "ns", "docker", idOther[:8]+strings.Repeat("d", 56)) + if err := os.MkdirAll(dfDir, 0o755); err != nil { + t.Fatal(err) + } + _ = os.WriteFile(filepath.Join(dfDir, "memory.events"), []byte("oom_kill 2\n"), 0o444) + + got, avail, err := readGuestOOMKills(root, 100) + if err != nil || !avail { + t.Fatalf("read: avail=%v err=%v", avail, err) + } + want := map[string]int64{idA: 3, idB: 0, idOther[:8] + strings.Repeat("d", 56): 2} + if len(got) != len(want) { + t.Fatalf("got %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Fatalf("got %v, want %v", got, want) + } + } + if _, leaked := got[idOther]; leaked { + t.Fatal("another guest's container leaked into this guest's answer") + } + // A guest with no cgroup folder (a VM, or not running): not available, no error. + if _, avail, err := readGuestOOMKills(root, 300); avail || err != nil { + t.Fatalf("a guest without a cgroup folder: avail=%v err=%v", avail, err) + } +} + +// Through the route, with the guest's own token. +func TestR528_TheRouteAnswersTheTokensGuest(t *testing.T) { + root := t.TempDir() + fakeCgroup(t, root, "8200", idA, "oom_kill 5\n") + fakeCgroup(t, root, "9300", idB, "oom_kill 1\n") // token B's guest: never in A's answer + srv := newTestServerS(t, &fakeGuests{}, &fakeBackups{}, &fakeStore{}, nil) + srv.cgroupRoot = root + w := do(t, srv.Handler(), "GET", "/guest/oom-kills", "A", "") + if w.Code != http.StatusOK { + t.Fatalf("route: %d %s", w.Code, w.Body.String()) + } + var env struct { + Data struct { + VMID int `json:"vmid"` + Available bool `json:"available"` + Containers map[string]int64 `json:"containers"` + } `json:"data"` + } + if err := json.Unmarshal(w.Body.Bytes(), &env); err != nil { + t.Fatal(err) + } + if env.Data.VMID != 8200 || !env.Data.Available || len(env.Data.Containers) != 1 || env.Data.Containers[idA] != 5 { + t.Fatalf("answer %s — want guest 8200's one container with 5", w.Body.String()) + } + if w := do(t, srv.Handler(), "GET", "/guest/oom-kills?vmid=9300", "A", ""); w.Code != http.StatusForbidden { + t.Fatalf("a cross-guest ask: %d, want 403", w.Code) + } + if w := do(t, srv.Handler(), "GET", "/guest/oom-kills", "", ""); w.Code != http.StatusUnauthorized { + t.Fatalf("no token: %d", w.Code) + } +} diff --git a/internal/localapi/server.go b/internal/localapi/server.go index c957b4e..d2421be 100644 --- a/internal/localapi/server.go +++ b/internal/localapi/server.go @@ -459,6 +459,8 @@ type Server struct { escrowLookPath func(file string) (string, error) // statFile reports whether a path exists (preflight's staged-secret item; tests inject). statFile func(path string) bool + // cgroupRoot is where the host's cgroup tree is mounted (R-528, GET /guest/oom-kills); "" = /sys/fs/cgroup. Tests set it. + cgroupRoot string baseCtx context.Context // for fire-and-forget backups; set in Run } @@ -576,6 +578,9 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /host/crash-guard", s.withGuest(s.handleCrashGuard)) // R-1247: the weekly upload line test's last measurement — the controller's qBittorrent limit and upload watch // read it. Read-only; never measured (or the job not wired) = present:false. + // R-528: the kernel's OOM-kill counter of each of the guest's containers, read on the host (Docker's own flag and + // event can stay silent inside an LXC guest). Read-only; the asking guest's containers only. + mux.HandleFunc("GET /guest/oom-kills", s.withGuest(s.handleGuestOOMKills)) mux.HandleFunc("GET /host/line-test", s.withGuest(s.handleLineTest)) mux.HandleFunc("POST /host/line-test/run", s.withGuest(s.handleLineTestRun)) // R-1250: a test on request // Disk management (slice 8C) — self-scoped; format routes through the data-bearing classifier+gate.