R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)
gates / gates (push) Successful in 32s

A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the
details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only.
Tests TestR528_* red first / red-proved. Commit only.
This commit is contained in:
admin committed 2026-10-28 23:22:46 +01:00
1 parent e33cffaee1
commit eed23e7594
9 files changed
+297 -7

No files matched your search

+31
View File
@@ -0,0 +1,31 @@
package agentapi
import (
"context"
"encoding/json"
"fmt"
)
// GuestOOMKillsState mirrors the agent's GET /guest/oom-kills (R-528, agent ≥ the R-528 commit): the kernel's oom_kill
// counter of each of THIS guest's containers, read on the host — Docker's own OOMKilled flag can stay false inside an
// LXC guest. Containers maps the full 64-hex container id to the count for the container's current run.
// Available=false: the guest has no cgroup folder the host can read. An agent that predates the route answers 404 (a
// *StatusError) — the controller then keeps Docker's flag only.
type GuestOOMKillsState struct {
VMID int `json:"vmid"`
Available bool `json:"available"`
Containers map[string]int64 `json:"containers"`
}
// GuestOOMKills calls GET /guest/oom-kills.
func (c *Client) GuestOOMKills(ctx context.Context) (GuestOOMKillsState, error) {
var out GuestOOMKillsState
body, err := c.get(ctx, "/guest/oom-kills")
if err != nil {
return out, err
}
if err := json.Unmarshal(body, &out); err != nil {
return out, fmt.Errorf("agentapi: decode /guest/oom-kills: %w", err)
}
return out, nil
}
+9 -1
View File
@@ -163,6 +163,8 @@ type UpdateDetails struct {
type AppDetails struct {
StackName string `json:"stack_name,omitempty"`
DisplayName string `json:"display_name,omitempty"`
// Source (app_oom only, R-528): what saw the kill — "docker" (State.OOMKilled) or "cgroup" (the host's kernel counter).
Source string `json:"source,omitempty"`
}
// CrossDriveDetails holds structured data for cross-drive backup events.
@@ -804,6 +806,12 @@ func (n *Notifier) NotifyAppStartFailures(apps []AppRunState) {
// that stays OOM-marked does not repeat. "warning" — the hub vocabulary (R-329). Operator-only
// hub-side (hub >= v0.114.0 registers it): the household sees the dashboard tag.
func (n *Notifier) NotifyAppOOM(stack, container, startedAt string, kills int64, memLimit, peak string) {
n.NotifyAppOOMFrom(stack, container, startedAt, kills, memLimit, peak, "")
}
// NotifyAppOOMFrom is NotifyAppOOM with the source that saw the kill (R-528: "docker" or "cgroup"), carried in the
// event's details. One event per container run, whichever source saw it.
func (n *Notifier) NotifyAppOOMFrom(stack, container, startedAt string, kills int64, memLimit, peak, source string) {
key := container + "|" + startedAt
n.mu.Lock()
if n.oomSeen == nil {
@@ -816,7 +824,7 @@ func (n *Notifier) NotifyAppOOM(stack, container, startedAt string, kills int64,
if first {
n.emit("app_oom", "warning",
fmt.Sprintf("Alkalmazás memóriája elfogyott: %s (%s) — egy folyamatát a memóriakorlát leállította", stack, container),
AppDetails{StackName: stack, DisplayName: container})
AppDetails{StackName: stack, DisplayName: container, Source: source})
}
if storm {
n.logger.Printf("[ERROR] [notify] %s: container %s OOM STORM — %d kills in %d min (limit %s, peak %s)",
+2
View File
@@ -284,6 +284,8 @@ type Manager struct {
// R-514: last OOM scan (oom.go), for the dashboard.
oomMu sync.Mutex
oomCache map[string][]string
// hostOOMKills (R-528) is the host agent's per-container OOM counter; nil = Docker's flag only (SetHostOOMKills).
hostOOMKills func() map[string]int64
// R-513: FileBrowser admin-password seams (filebrowser_password.go); nil/"" in production.
fbHTTP fbHTTPDo
fbBaseURL string
+47 -4
View File
@@ -27,6 +27,21 @@ type OOMContainer struct {
Kills int64
// MemLimit / Peak are the cgroup's memory.max and memory.peak, as "<n>M" (or "max"); "" unread.
MemLimit, Peak string
// Source names what flagged it: OOMSourceDocker (State.OOMKilled), OOMSourceCgroup (R-528 — only the host's
// per-container kernel counter saw it; Docker's flag stayed false), or "docker,cgroup" (both).
Source string
}
const (
OOMSourceDocker = "docker"
OOMSourceCgroup = "cgroup"
)
// SetHostOOMKills wires the host-side OOM counter (R-528): container id → the kernel's oom_kill for the container's
// CURRENT run (its cgroup scope is new at each start), read by the host's agent. nil, or a nil answer (no agent, an
// agent without the route, a failed call) = Docker's flag only, as before. INIT-ONLY (main.go).
func (m *Manager) SetHostOOMKills(fn func() map[string]int64) {
m.hostOOMKills = fn
}
// ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
@@ -52,20 +67,45 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
return nil, nil
}
sort.Strings(names)
args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...)
args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}|{{.Id}}"}, names...)
out, err := m.execCommand("docker", args...)
if err != nil && strings.TrimSpace(out) == "" {
return nil, err
}
// R-528: the host's counter, asked once per scan (an HTTPS call to the agent); nil = Docker's flag only.
var host map[string]int64
if m.hostOOMKills != nil {
host = m.hostOOMKills()
}
var found []OOMContainer
hostKills := map[int]int64{}
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.SplitN(strings.TrimSpace(line), "|", 3)
if len(f) != 3 || f[1] != "true" {
f := strings.SplitN(strings.TrimSpace(line), "|", 4)
if len(f) < 3 {
continue
}
id := ""
if len(f) == 4 {
id = strings.TrimSpace(f[3])
}
hk, counted := host[id]
source := ""
switch {
case f[1] == "true" && id != "" && counted && hk > 0:
source = OOMSourceDocker + "," + OOMSourceCgroup // both saw it
case f[1] == "true":
source = OOMSourceDocker
case id != "" && counted && hk > 0:
source = OOMSourceCgroup
default:
continue
}
cname := strings.TrimPrefix(f[0], "/")
if st, ok := owner[cname]; ok {
found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1})
if counted && id != "" {
hostKills[len(found)] = hk
}
found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2], Kills: -1, Source: source})
}
}
// R-636: only for the containers already flagged — one `docker exec` each, and the flagged set is
@@ -75,6 +115,9 @@ func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
out, _ := m.execCommand("docker", "exec", found[i].Container, "cat",
"/sys/fs/cgroup/memory.events", "/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory.peak")
found[i].Kills, found[i].MemLimit, found[i].Peak = parseCgroupMemory(out)
if hk, ok := hostKills[i]; ok && hk > found[i].Kills {
found[i].Kills = hk // the host's count is the kernel's own; the in-container read can be -1 (no cat)
}
}
m.setOOMCache(found)
return found, nil
@@ -0,0 +1,84 @@
package stacks
import (
"io"
"log"
"strings"
"testing"
)
// R-528 — inside a Felhom LXC guest Docker can leave State.OOMKilled false after a memory-limit kill (measured
// 2026-09-15/16). The host's agent reads the kernel's own per-container counter (GET /guest/oom-kills); a counter above
// 0 for the RUNNING container's id is an OOM kill in this run, whatever Docker says. The scan flags it with
// Source "cgroup" and the host's count, and the existing app_oom line (once per run) fires for it.
const r528ID = "4f5e6d7c8b9a0f1e2d3c4b5a69788796a5b4c3d2e1f00112233445566778899a"
// RED-PROOF: ignore the host counter in ScanOOMKilled (only `OOMKilled == true` flags) → the killed container is not
// seen → FAILS.
func TestR528_AHostCounterAboveZeroIsAnOOMKillWhateverDockerSays(t *testing.T) {
var execd []string
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
"hog": {Name: "hog", Deployed: true, Containers: []ContainerInfo{{Name: "hog", State: StateRunning}}},
"bookstack": {Name: "bookstack", Deployed: true, Containers: []ContainerInfo{{Name: "bookstack", State: StateRunning}}},
}}
m.execFn = func(name string, args ...string) (string, error) {
if args[0] == "exec" {
execd = append(execd, args[1])
return "oom_kill 1\n67108864\n67100000\n", nil
}
if !strings.Contains(strings.Join(args, " "), "{{.Id}}") {
t.Errorf("the inspect must ask for the container id: %v", args)
}
return "/bookstack|false|t0|" + strings.Repeat("0", 64) + "\n/hog|false|2026-10-28T23:00:00Z|" + r528ID + "\n", nil
}
m.SetHostOOMKills(func() map[string]int64 {
return map[string]int64{r528ID: 2, strings.Repeat("0", 64): 0}
})
ooms, err := m.ScanOOMKilled()
if err != nil {
t.Fatal(err)
}
if len(ooms) != 1 || ooms[0].Stack != "hog" || ooms[0].Container != "hog" {
t.Fatalf("the host-counted kill was not seen: %+v", ooms)
}
if ooms[0].Source != OOMSourceCgroup || ooms[0].Kills != 2 || ooms[0].StartedAt != "2026-10-28T23:00:00Z" {
t.Fatalf("want source cgroup, the host's count 2 and the run's start: %+v", ooms[0])
}
if ooms[0].MemLimit != "64M" {
t.Errorf("limit/peak are still read for the flagged container: %+v", ooms[0])
}
if len(execd) != 1 || execd[0] != "hog" {
t.Errorf("exec must reach the flagged container only, got %v", execd)
}
}
// Docker's flag alone still flags (Source docker); no host source (an old agent, 9202 without an agent) = today's
// behaviour; a host counter of 0 never flags.
func TestR528_WithoutAHostCounterTheScanIsAsBefore(t *testing.T) {
m := &Manager{logger: log.New(io.Discard, "", 0), stacks: map[string]*Stack{
"a": {Name: "a", Deployed: true, Containers: []ContainerInfo{{Name: "a", State: StateRunning}}},
"b": {Name: "b", Deployed: true, Containers: []ContainerInfo{{Name: "b", State: StateRunning}}},
}}
m.execFn = func(name string, args ...string) (string, error) {
if args[0] == "exec" {
return "oom_kill 3\n", nil
}
return "/a|true|s1|" + strings.Repeat("a", 64) + "\n/b|false|s2|" + strings.Repeat("b", 64) + "\n", nil
}
ooms, _ := m.ScanOOMKilled()
if len(ooms) != 1 || ooms[0].Stack != "a" || ooms[0].Source != OOMSourceDocker || ooms[0].Kills != 3 {
t.Fatalf("no host source: %+v", ooms)
}
m.SetHostOOMKills(func() map[string]int64 { return map[string]int64{strings.Repeat("b", 64): 0} })
ooms, _ = m.ScanOOMKilled()
if len(ooms) != 1 || ooms[0].Stack != "a" {
t.Fatalf("a host counter of 0 flagged a container: %+v", ooms)
}
// Both saw it: the source names both, the host's count wins over a lower in-container read.
m.SetHostOOMKills(func() map[string]int64 { return map[string]int64{strings.Repeat("a", 64): 7} })
ooms, _ = m.ScanOOMKilled()
if len(ooms) != 1 || ooms[0].Source != OOMSourceDocker+","+OOMSourceCgroup || ooms[0].Kills != 7 {
t.Fatalf("both sources: %+v", ooms)
}
}