felhom-controller/controller/internal/stacks/oom.go

package stacks

import (
	"sort"
	"strings"
)

// R-514 (v0.243.0) — an OOM-killed worker inside a RUNNING container is invisible to the state model.
//
// BIGNIGHT: Paperless's celery worker was killed by the memory cgroup inside the webserver container;
// the container kept running (`docker inspect` → OOMKilled=true, RestartCount=0), the app read „Fut",
// 11 documents failed and 8 waited forever. Docker's State.OOMKilled is set when ANY process in the
// container's cgroup was OOM-killed and stays set until the container restarts — so it is the
// observable, read for the running containers of deployed stacks.

// OOMContainer is one container whose cgroup had a process OOM-killed since it started.
type OOMContainer struct {
	Stack     string
	Container string
	StartedAt string // identifies the container run — one event per run
}

// ScanOOMKilled inspects the containers of every deployed, running-ish stack in ONE docker call and
// returns those with State.OOMKilled=true. It also caches the per-stack result for the dashboard.
func (m *Manager) ScanOOMKilled() ([]OOMContainer, error) {
	m.mu.RLock()
	owner := map[string]string{}
	var names []string
	for name, st := range m.stacks {
		if !st.Deployed {
			continue
		}
		for _, c := range st.Containers {
			if c.State == StateRunning || c.State == StateUnhealthy || c.State == StateRestarting {
				owner[c.Name] = name
				names = append(names, c.Name)
			}
		}
	}
	m.mu.RUnlock()
	if len(names) == 0 {
		m.setOOMCache(nil)
		return nil, nil
	}
	sort.Strings(names)
	args := append([]string{"inspect", "-f", "{{.Name}}|{{.State.OOMKilled}}|{{.State.StartedAt}}"}, names...)
	out, err := m.execCommand("docker", args...)
	if err != nil && strings.TrimSpace(out) == "" {
		return nil, err
	}
	var found []OOMContainer
	for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
		f := strings.SplitN(strings.TrimSpace(line), "|", 3)
		if len(f) != 3 || f[1] != "true" {
			continue
		}
		cname := strings.TrimPrefix(f[0], "/")
		if st, ok := owner[cname]; ok {
			found = append(found, OOMContainer{Stack: st, Container: cname, StartedAt: f[2]})
		}
	}
	m.setOOMCache(found)
	return found, nil
}

func (m *Manager) setOOMCache(found []OOMContainer) {
	cache := map[string][]string{}
	for _, o := range found {
		cache[o.Stack] = append(cache[o.Stack], o.Container)
	}
	m.oomMu.Lock()
	m.oomCache = cache
	m.oomMu.Unlock()
}

// OOMKilledStacks returns stack name → OOM-killed container names from the last scan.
func (m *Manager) OOMKilledStacks() map[string][]string {
	m.oomMu.Lock()
	defer m.oomMu.Unlock()
	out := make(map[string][]string, len(m.oomCache))
	for k, v := range m.oomCache {
		out[k] = append([]string(nil), v...)
	}
	return out
}