R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)
gates / gates (push) Successful in 32s
gates / gates (push) Successful in 32s
A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only. Tests TestR528_* red first / red-proved. Commit only.
This commit is contained in:
1 parent
e33cffaee1
commit
eed23e7594
9 files changed
+297
-7
No files matched your search
@@ -27,6 +27,21 @@ type OOMContainer struct {
|
||||
Kills int64
|
||||
// MemLimit / Peak are the cgroup's memory.max and memory.peak, as "<n>M" (or "max"); "" unread.
|
||||
MemLimit, Peak string
|
||||
// Source names what flagged it: OOMSourceDocker (State.OOMKilled), OOMSourceCgroup (R-528 — only the host's
|
||||
// per-container kernel counter saw it; Docker's flag stayed false), or "docker,cgroup" (both).
|
||||
Source string
|
||||
}
|
||||
|
||||
const (
|
||||
OOMSourceDocker = "docker"
|
||||
OOMSourceCgroup = "cgroup"
|
||||
)
|
||||
|
||||
// SetHostOOMKills wires the host-side OOM counter (R-528): container id → the kernel's oom_kill for the container's
|
||||
// CURRENT run (its cgroup scope is new at each start), read by the host's agent. nil, or a nil answer (no agent, an
|
||||
// agent without the route, a failed call) = Docker's flag only, as before. INIT-ONLY (main.go).
|
||||
func (m *Manager) SetHostOOMKills(fn func() map[string]int64) {
|
||||
m.hostOOMKills = fn
|
||||
}
|
||||
|
||||
// ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
|
||||
@@ -52,20 +67,45 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
|
||||
return nil, nil
|
||||
}
|
||||
sort.Strings(names)
|
||||
args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...)
|
||||
args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}|{{.Id}}"}, names...)
|
||||
out, err := m.execCommand("docker", args...)
|
||||
if err != nil && strings.TrimSpace(out) == "" {
|
||||
return nil, err
|
||||
}
|
||||
// R-528: the host's counter, asked once per scan (an HTTPS call to the agent); nil = Docker's flag only.
|
||||
var host map[string]int64
|
||||
if m.hostOOMKills != nil {
|
||||
host = m.hostOOMKills()
|
||||
}
|
||||
var found []OOMContainer
|
||||
hostKills := map[int]int64{}
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
f := strings.SplitN(strings.TrimSpace(line), "|", 3)
|
||||
if len(f) != 3 || f[1] != "true" {
|
||||
f := strings.SplitN(strings.TrimSpace(line), "|", 4)
|
||||
if len(f) < 3 {
|
||||
continue
|
||||
}
|
||||
id := ""
|
||||
if len(f) == 4 {
|
||||
id = strings.TrimSpace(f[3])
|
||||
}
|
||||
hk, counted := host[id]
|
||||
source := ""
|
||||
switch {
|
||||
case f[1] == "true" && id != "" && counted && hk > 0:
|
||||
source = OOMSourceDocker + "," + OOMSourceCgroup // both saw it
|
||||
case f[1] == "true":
|
||||
source = OOMSourceDocker
|
||||
case id != "" && counted && hk > 0:
|
||||
source = OOMSourceCgroup
|
||||
default:
|
||||
continue
|
||||
}
|
||||
cname := strings.TrimPrefix(f[0], "/")
|
||||
if st, ok := owner[cname]; ok {
|
||||
found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1})
|
||||
if counted && id != "" {
|
||||
hostKills[len(found)] = hk
|
||||
}
|
||||
found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1, Source: source})
|
||||
}
|
||||
}
|
||||
// R-636: only for the containers already flagged — one `docker exec` each, and the flagged set is
|
||||
@@ -75,6 +115,9 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
|
||||
out, _ := m.execCommand("docker", "exec", found[i].Container, "cat",
|
||||
"/sys/fs/cgroup/memory.events", "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.peak")
|
||||
found[i].Kills, found[i].MemLimit, found[i].Peak = parseCgroupMemory(out)
|
||||
if hk, ok := hostKills[i]; ok && hk > found[i].Kills {
|
||||
found[i].Kills = hk // the host's count is the kernel's own; the in-container read can be -1 (no cat)
|
||||
}
|
||||
}
|
||||
m.setOOMCache(found)
|
||||
return found, nil
|
||||
|
||||
Reference in new issue
Block a user