R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)
gates / gates (push) Successful in 32s

A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the
details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only.
Tests TestR528_* red first / red-proved. Commit only.
This commit is contained in:
admin committed 2026-10-28 23:22:46 +01:00
1 parent e33cffaee1
commit eed23e7594
9 files changed
+297 -7

No files matched your search

+47 -4
View File
@@ -27,6 +27,21 @@ type OOMContainer struct {
Kills int64
// MemLimit / Peak are the cgroup's memory.max and memory.peak, as "<n>M" (or "max"); "" unread.
MemLimit, Peak string
// Source names what flagged it: OOMSourceDocker (State.OOMKilled), OOMSourceCgroup (R-528 — only the host's
// per-container kernel counter saw it; Docker's flag stayed false), or "docker,cgroup" (both).
Source string
}
const (
OOMSourceDocker = "docker"
OOMSourceCgroup = "cgroup"
)
// SetHostOOMKills wires the host-side OOM counter (R-528): container id → the kernel's oom_kill for the container's
// CURRENT run (its cgroup scope is new at each start), read by the host's agent. nil, or a nil answer (no agent, an
// agent without the route, a failed call) = Docker's flag only, as before. INIT-ONLY (main.go).
func (m *Manager) SetHostOOMKills(fn func() map[string]int64) {
m.hostOOMKills = fn
}
// ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
@@ -52,20 +67,45 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
return nil, nil
}
sort.Strings(names)
args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...)
args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}|{{.Id}}"}, names...)
out, err := m.execCommand("docker", args...)
if err != nil && strings.TrimSpace(out) == "" {
return nil, err
}
// R-528: the host's counter, asked once per scan (an HTTPS call to the agent); nil = Docker's flag only.
var host map[string]int64
if m.hostOOMKills != nil {
host = m.hostOOMKills()
}
var found []OOMContainer
hostKills := map[int]int64{}
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.SplitN(strings.TrimSpace(line), "|", 3)
if len(f) != 3 || f[1] != "true" {
f := strings.SplitN(strings.TrimSpace(line), "|", 4)
if len(f) < 3 {
continue
}
id := ""
if len(f) == 4 {
id = strings.TrimSpace(f[3])
}
hk, counted := host[id]
source := ""
switch {
case f[1] == "true" && id != "" && counted && hk > 0:
source = OOMSourceDocker + "," + OOMSourceCgroup // both saw it
case f[1] == "true":
source = OOMSourceDocker
case id != "" && counted && hk > 0:
source = OOMSourceCgroup
default:
continue
}
cname := strings.TrimPrefix(f[0], "/")
if st, ok := owner[cname]; ok {
found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1})
if counted && id != "" {
hostKills[len(found)] = hk
}
found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1, Source: source})
}
}
// R-636: only for the containers already flagged — one `docker exec` each, and the flagged set is
@@ -75,6 +115,9 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
out, _ := m.execCommand("docker", "exec", found[i].Container, "cat",
"/sys/fs/cgroup/memory.events", "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.peak")
found[i].Kills, found[i].MemLimit, found[i].Peak = parseCgroupMemory(out)
if hk, ok := hostKills[i]; ok && hk > found[i].Kills {
found[i].Kills = hk // the host's count is the kernel's own; the in-container read can be -1 (no cat)
}
}
m.setOOMCache(found)
return found, nil