package stacks import ( "sort" "strings" ) // R-514 (v0.243.0) — an OOM-killed worker inside a RUNNING container is invisible to the state model. // // BIGNIGHT: Paperless's celery worker was killed by the memory cgroup inside the webserver container; // the container kept running (`docker inspect` → OOMKilled=true, RestartCount=0), the app read „Fut", // 11 documents failed and 8 waited forever. Docker's State.OOMKilled is set when ANY process in the // container's cgroup was OOM-killed and stays set until the container restarts — so it is the // observable, read for the running containers of deployed stacks. // OOMContainer is one container whose cgroup had a process OOM-killed since it started. type OOMContainer struct { Stack string Container string StartedAt string // identifies the container run — one event per run } // ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and // returns those with State.OOMKilled=true. It also caches the per-stack result for the dashboard. func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) { m.mu.RLock() owner := map[string]string{} var names []string for name, st := range m.stacks { if !st.Deployed { continue } for _, c := range st.Containers { if c.State == StateRunning || c.State == StateUnhealthy || c.State == StateRestarting { owner[c.Name] = name names = append(names, c.Name) } } } m.mu.RUnlock() if len(names) == 0 { m.setOOMCache(nil) return nil, nil } sort.Strings(names) args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...) out, err := m.execCommand("docker", args...) if err != nil && strings.TrimSpace(out) == "" { return nil, err } var found []OOMContainer for _, line := range strings.Split(strings.TrimSpace(out), "\n") { f := strings.SplitN(strings.TrimSpace(line), "|", 3) if len(f) != 3 || f[1] != "true" { continue } cname := strings.TrimPrefix(f[0], "/") if st, ok := owner[cname]; ok { found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2]}) } } m.setOOMCache(found) return found, nil } func (m *Manager) setOOMCache(found []OOMContainer) { cache := map[string][]string{} for _, o := range found { cache[o.Stack] = append(cache[o.Stack], o.Container) } m.oomMu.Lock() m.oomCache = cache m.oomMu.Unlock() } // OOMKilledStacks returns stack name → OOM-killed container names from the last scan. func (m *Manager) OOMKilledStacks() map[string][]string { m.oomMu.Lock() defer m.oomMu.Unlock() out := make(map[string][]string, len(m.oomCache)) for k, v := range m.oomCache { out[k] = append([]string(nil), v...) } return out }