R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)

A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only. Tests TestR528_* red first / red-proved. Commit only.
2026-10-28 23:22:46 +01:00
1 parent e33cffaee1
commit eed23e7594
@@ -284,6 +284,8 @@ type Manager struct {
 	// R-514: last OOM scan (oom.go), for the dashboard.
 	oomMu    sync.Mutex
 	oomCache map[string][]string
+	// hostOOMKills (R-528) is the host agent's per-container OOM counter; nil = Docker's flag only (SetHostOOMKills).
+	hostOOMKills func() map[string]int64
 	// R-513: FileBrowser admin-password seams (filebrowser_password.go); nil/"" in production.
 	fbHTTP    fbHTTPDo
 	fbBaseURL string
@@ -27,6 +27,21 @@ type OOMContainer struct {
 	Kills int64
 	// MemLimit / Peak are the cgroup's memory.max and memory.peak, as "<n>M" (or "max"); "" unread.
 	MemLimit, Peak string
+	// Source names what flagged it: OOMSourceDocker (State.OOMKilled), OOMSourceCgroup (R-528 — only the host's
+	// per-container kernel counter saw it; Docker's flag stayed false), or "docker,cgroup" (both).
+	Source string
+}
+
+const (
+	OOMSourceDocker = "docker"
+	OOMSourceCgroup = "cgroup"
+)
+
+// SetHostOOMKills wires the host-side OOM counter (R-528): container id → the kernel's oom_kill for the container's
+// CURRENT run (its cgroup scope is new at each start), read by the host's agent. nil, or a nil answer (no agent, an
+// agent without the route, a failed call) = Docker's flag only, as before. INIT-ONLY (main.go).
+func (m *Manager) SetHostOOMKills(fn func() map[string]int64) {
+	m.hostOOMKills = fn
 }

 // ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
@@ -52,20 +67,45 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
 		return nil, nil
 	}
 	sort.Strings(names)
-	args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...)
+	args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}|{{.Id}}"}, names...)
 	out, err := m.execCommand("docker", args...)
 	if err != nil && strings.TrimSpace(out) == "" {
 		return nil, err
 	}
+	// R-528: the host's counter, asked once per scan (an HTTPS call to the agent); nil = Docker's flag only.
+	var host map[string]int64
+	if m.hostOOMKills != nil {
+		host = m.hostOOMKills()
+	}
 	var found []OOMContainer
+	hostKills := map[int]int64{}
 	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
-		f := strings.SplitN(strings.TrimSpace(line), "|", 3)
-		if len(f) != 3 || f[1] != "true" {
+		f := strings.SplitN(strings.TrimSpace(line), "|", 4)
+		if len(f) < 3 {
+			continue
+		}
+		id := ""
+		if len(f) == 4 {
+			id = strings.TrimSpace(f[3])
+		}
+		hk, counted := host[id]
+		source := ""
+		switch {
+		case f[1] == "true" && id != "" && counted && hk > 0:
+			source = OOMSourceDocker + "," + OOMSourceCgroup // both saw it
+		case f[1] == "true":
+			source = OOMSourceDocker
+		case id != "" && counted && hk > 0:
+			source = OOMSourceCgroup
+		default:
 			continue
 		}
 		cname := strings.TrimPrefix(f[0], "/")
 		if st, ok := owner[cname]; ok {
-			found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1})
+			if counted && id != "" {
+				hostKills[len(found)] = hk
+			}
+			found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1, Source: source})
 		}
 	}
 	// R-636: only for the containers already flagged — one `docker exec` each, and the flagged set is
@@ -75,6 +115,9 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
 		out, _ := m.execCommand("docker", "exec", found[i].Container, "cat",
 			"/sys/fs/cgroup/memory.events", "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.peak")
 		found[i].Kills, found[i].MemLimit, found[i].Peak = parseCgroupMemory(out)
+		if hk, ok := hostKills[i]; ok && hk > found[i].Kills {
+			found[i].Kills = hk // the host's count is the kernel's own; the in-container read can be -1 (no cat)
+		}
 	}
 	m.setOOMCache(found)
 	return found, nil
@@ -0,0 +1,84 @@
+package stacks
+
+import (
+	"io"
+	"log"
+	"strings"
+	"testing"
+)
+
+// R-528 — inside a Felhom LXC guest Docker can leave State.OOMKilled false after a memory-limit kill (measured
+// 2026-09-15/16). The host's agent reads the kernel's own per-container counter (GET /guest/oom-kills); a counter above
+// 0 for the RUNNING container's id is an OOM kill in this run, whatever Docker says. The scan flags it with
+// Source "cgroup" and the host's count, and the existing app_oom line (once per run) fires for it.
+
+const r528ID = "4f5e6d7c8b9a0f1e2d3c4b5a69788796a5b4c3d2e1f00112233445566778899a"
+
+// RED-PROOF: ignore the host counter in ScanOOMKilled (only `OOMKilled == true` flags) → the killed container is not
+// seen → FAILS.
+func TestR528_AHostCounterAboveZeroIsAnOOMKillWhateverDockerSays(t *testing.T) {
+	var execd []string
+	m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
+		"hog":       {Name: "hog", Deployed: true, Containers: []ContainerInfo{{Name: "hog", State: StateRunning}}},
+		"bookstack": {Name: "bookstack", Deployed: true, Containers: []ContainerInfo{{Name: "bookstack", State: StateRunning}}},
+	}}
+	m.execFn = func(name string, args ...string) (string, error) {
+		if args[0] == "exec" {
+			execd = append(execd, args[1])
+			return "oom_kill 1\n67108864\n67100000\n", nil
+		}
+		if !strings.Contains(strings.Join(args, " "), "{{.Id}}") {
+			t.Errorf("the inspect must ask for the container id: %v", args)
+		}
+		return "/bookstack|false|t0|" + strings.Repeat("0", 64) + "\n/hog|false|2026-10-28T23:00:00Z|" + r528ID + "\n", nil
+	}
+	m.SetHostOOMKills(func() map[string]int64 {
+		return map[string]int64{r528ID: 2, strings.Repeat("0", 64): 0}
+	})
+	ooms, err := m.ScanOOMKilled()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(ooms) != 1 || ooms[0].Stack != "hog" || ooms[0].Container != "hog" {
+		t.Fatalf("the host-counted kill was not seen: %+v", ooms)
+	}
+	if ooms[0].Source != OOMSourceCgroup || ooms[0].Kills != 2 || ooms[0].StartedAt != "2026-10-28T23:00:00Z" {
+		t.Fatalf("want source cgroup, the host's count 2 and the run's start: %+v", ooms[0])
+	}
+	if ooms[0].MemLimit != "64M" {
+		t.Errorf("limit/peak are still read for the flagged container: %+v", ooms[0])
+	}
+	if len(execd) != 1 || execd[0] != "hog" {
+		t.Errorf("exec must reach the flagged container only, got %v", execd)
+	}
+}
+
+// Docker's flag alone still flags (Source docker); no host source (an old agent, 9202 without an agent) = today's
+// behaviour; a host counter of 0 never flags.
+func TestR528_WithoutAHostCounterTheScanIsAsBefore(t *testing.T) {
+	m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
+		"a": {Name: "a", Deployed: true, Containers: []ContainerInfo{{Name: "a", State: StateRunning}}},
+		"b": {Name: "b", Deployed: true, Containers: []ContainerInfo{{Name: "b", State: StateRunning}}},
+	}}
+	m.execFn = func(name string, args ...string) (string, error) {
+		if args[0] == "exec" {
+			return "oom_kill 3\n", nil
+		}
+		return "/a|true|s1|" + strings.Repeat("a", 64) + "\n/b|false|s2|" + strings.Repeat("b", 64) + "\n", nil
+	}
+	ooms, _ := m.ScanOOMKilled()
+	if len(ooms) != 1 || ooms[0].Stack != "a" || ooms[0].Source != OOMSourceDocker || ooms[0].Kills != 3 {
+		t.Fatalf("no host source: %+v", ooms)
+	}
+	m.SetHostOOMKills(func() map[string]int64 { return map[string]int64{strings.Repeat("b", 64): 0} })
+	ooms, _ = m.ScanOOMKilled()
+	if len(ooms) != 1 || ooms[0].Stack != "a" {
+		t.Fatalf("a host counter of 0 flagged a container: %+v", ooms)
+	}
+	// Both saw it: the source names both, the host's count wins over a lower in-container read.
+	m.SetHostOOMKills(func() map[string]int64 { return map[string]int64{strings.Repeat("a", 64): 7} })
+	ooms, _ = m.ScanOOMKilled()
+	if len(ooms) != 1 || ooms[0].Source != OOMSourceDocker+","+OOMSourceCgroup || ooms[0].Kills != 7 {
+		t.Fatalf("both sources: %+v", ooms)
+	}
+}