R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)
gates / gates (push) Successful in 32s
gates / gates (push) Successful in 32s
A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only. Tests TestR528_* red first / red-proved. Commit only.
This commit is contained in:
1 parent
e33cffaee1
commit
eed23e7594
9 files changed
+297
-7
No files matched your search
@@ -0,0 +1,84 @@
|
||||
package stacks
|
||||
|
||||
import (
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// R-528 — inside a Felhom LXC guest Docker can leave State.OOMKilled false after a memory-limit kill (measured
|
||||
// 2026-09-15/16). The host's agent reads the kernel's own per-container counter (GET /guest/oom-kills); a counter above
|
||||
// 0 for the RUNNING container's id is an OOM kill in this run, whatever Docker says. The scan flags it with
|
||||
// Source "cgroup" and the host's count, and the existing app_oom line (once per run) fires for it.
|
||||
|
||||
const r528ID = "4f5e6d7c8b9a0f1e2d3c4b5a69788796a5b4c3d2e1f00112233445566778899a"
|
||||
|
||||
// RED-PROOF: ignore the host counter in ScanOOMKilled (only `OOMKilled == true` flags) → the killed container is not
|
||||
// seen → FAILS.
|
||||
func TestR528_AHostCounterAboveZeroIsAnOOMKillWhateverDockerSays(t *testing.T) {
|
||||
var execd []string
|
||||
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
|
||||
"hog": {Name: "hog", Deployed: true, Containers: []ContainerInfo{{Name: "hog", State: StateRunning}}},
|
||||
"bookstack": {Name: "bookstack", Deployed: true, Containers: []ContainerInfo{{Name: "bookstack", State: StateRunning}}},
|
||||
}}
|
||||
m.execFn = func(name string, args ...string) (string, error) {
|
||||
if args[0] == "exec" {
|
||||
execd = append(execd, args[1])
|
||||
return "oom_kill 1\n67108864\n67100000\n", nil
|
||||
}
|
||||
if !strings.Contains(strings.Join(args, " "), "{{.Id}}") {
|
||||
t.Errorf("the inspect must ask for the container id: %v", args)
|
||||
}
|
||||
return "/bookstack|false|t0|" + strings.Repeat("0", 64) + "\n/hog|false|2026-10-28T23:00:00Z|" + r528ID + "\n", nil
|
||||
}
|
||||
m.SetHostOOMKills(func() map[string]int64 {
|
||||
return map[string]int64{r528ID: 2, strings.Repeat("0", 64): 0}
|
||||
})
|
||||
ooms, err := m.ScanOOMKilled()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ooms) != 1 || ooms[0].Stack != "hog" || ooms[0].Container != "hog" {
|
||||
t.Fatalf("the host-counted kill was not seen: %+v", ooms)
|
||||
}
|
||||
if ooms[0].Source != OOMSourceCgroup || ooms[0].Kills != 2 || ooms[0].StartedAt != "2026-10-28T23:00:00Z" {
|
||||
t.Fatalf("want source cgroup, the host's count 2 and the run's start: %+v", ooms[0])
|
||||
}
|
||||
if ooms[0].MemLimit != "64M" {
|
||||
t.Errorf("limit/peak are still read for the flagged container: %+v", ooms[0])
|
||||
}
|
||||
if len(execd) != 1 || execd[0] != "hog" {
|
||||
t.Errorf("exec must reach the flagged container only, got %v", execd)
|
||||
}
|
||||
}
|
||||
|
||||
// Docker's flag alone still flags (Source docker); no host source (an old agent, 9202 without an agent) = today's
|
||||
// behaviour; a host counter of 0 never flags.
|
||||
func TestR528_WithoutAHostCounterTheScanIsAsBefore(t *testing.T) {
|
||||
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
|
||||
"a": {Name: "a", Deployed: true, Containers: []ContainerInfo{{Name: "a", State: StateRunning}}},
|
||||
"b": {Name: "b", Deployed: true, Containers: []ContainerInfo{{Name: "b", State: StateRunning}}},
|
||||
}}
|
||||
m.execFn = func(name string, args ...string) (string, error) {
|
||||
if args[0] == "exec" {
|
||||
return "oom_kill 3\n", nil
|
||||
}
|
||||
return "/a|true|s1|" + strings.Repeat("a", 64) + "\n/b|false|s2|" + strings.Repeat("b", 64) + "\n", nil
|
||||
}
|
||||
ooms, _ := m.ScanOOMKilled()
|
||||
if len(ooms) != 1 || ooms[0].Stack != "a" || ooms[0].Source != OOMSourceDocker || ooms[0].Kills != 3 {
|
||||
t.Fatalf("no host source: %+v", ooms)
|
||||
}
|
||||
m.SetHostOOMKills(func() map[string]int64 { return map[string]int64{strings.Repeat("b", 64): 0} })
|
||||
ooms, _ = m.ScanOOMKilled()
|
||||
if len(ooms) != 1 || ooms[0].Stack != "a" {
|
||||
t.Fatalf("a host counter of 0 flagged a container: %+v", ooms)
|
||||
}
|
||||
// Both saw it: the source names both, the host's count wins over a lower in-container read.
|
||||
m.SetHostOOMKills(func() map[string]int64 { return map[string]int64{strings.Repeat("a", 64): 7} })
|
||||
ooms, _ = m.ScanOOMKilled()
|
||||
if len(ooms) != 1 || ooms[0].Source != OOMSourceDocker+","+OOMSourceCgroup || ooms[0].Kills != 7 {
|
||||
t.Fatalf("both sources: %+v", ooms)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user