R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)
gates / gates (push) Successful in 32s

A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the
details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only.
Tests TestR528_* red first / red-proved. Commit only.
This commit is contained in:
admin committed 2026-10-28 23:22:46 +01:00
1 parent e33cffaee1
commit eed23e7594
9 files changed
+297 -7

No files matched your search

@@ -0,0 +1,50 @@
package main
import (
"context"
"errors"
"log"
"net/http"
"sync"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
)
// hostOOMSource (R-528) is the 30-s dead-app scan's host-side OOM counter: one agent call per scan (5 s cap). An agent
// without the route (404) is not asked again for hostOOMRetryAfter, and said once; any other failure is a nil answer
// for this scan only (Docker's flag still works). Pinned by TestR528_HostOOMSource*.
const hostOOMRetryAfter = 10 * time.Minute
func hostOOMSource(ask func(context.Context) (agentapi.GuestOOMKillsState, error), logger *log.Logger, now func() time.Time) func() map[string]int64 {
var mu sync.Mutex
var until time.Time
said := false
return func() map[string]int64 {
mu.Lock()
defer mu.Unlock()
if now().Before(until) {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
st, err := ask(ctx)
if err != nil {
var se *agentapi.StatusError
if errors.As(err, &se) && se.Code == http.StatusNotFound {
until = now().Add(hostOOMRetryAfter)
if !said {
logger.Printf("[INFO] [deadapp] the agent has no OOM counter route (R-528) — Docker's OOM flag only; asked again in %s", hostOOMRetryAfter)
said = true
}
return nil
}
logger.Printf("[WARN] [deadapp] the host OOM counter could not be read: %v — Docker's OOM flag only for this scan", err)
return nil
}
if !st.Available {
return nil
}
return st.Containers
}
}
+5 -2
View File
@@ -1035,6 +1035,9 @@ func main() {
logger.Printf("[WARN] [deadapp] agent client init failed (%v) — the boot grace cannot learn of a crash boot; 90 s", err)
} else {
crashProbe = crashboot.AgentProbe(ac.CrashGuard)
// R-528: the 30-s scan also asks the host for the kernel's per-container OOM counter (Docker's own flag can
// stay false inside an LXC guest). TestR528_HostOOMSourceIsWiredAtStartup.
stackMgr.SetHostOOMKills(hostOOMSource(ac.GuestOOMKills, logger, time.Now))
}
}
bootGrace := crashboot.New(startTime, crashProbe, logger)
@@ -1094,8 +1097,8 @@ func main() {
logger.Printf("[WARN] [deadapp] OOM scan failed: %v", oerr)
} else {
for _, o := range ooms {
logger.Printf("[WARN] [deadapp] %s: container %s was OOM-killed (started %s; kills %d, limit %s, peak %s)", o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak)
notifier.NotifyAppOOM(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak)
logger.Printf("[WARN] [deadapp] %s: container %s was OOM-killed (started %s; kills %d, limit %s, peak %s; seen by %s)", o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak, o.Source)
notifier.NotifyAppOOMFrom(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak, o.Source)
}
}
// v0.269.0 (`09` §3 decision 28, R-667): a crash loop or an OOM storm is STOPPED by the box.
@@ -0,0 +1,60 @@
package main
import (
"context"
"errors"
"io"
"log"
"net/http"
"testing"
"time"
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
)
// R-528: the scan's host counter source. An old agent (404) is asked again only after hostOOMRetryAfter; a transient
// failure is retried at the next scan; the counters pass through.
// RED-PROOF: drop the 404 back-off → an old agent is asked at every 30-s scan (calls 3, not 1) → FAILS.
func TestR528_HostOOMSourceBacksOffAnOldAgent(t *testing.T) {
clock := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
calls := 0
var answer error = &agentapi.StatusError{Path: "/guest/oom-kills", Code: http.StatusNotFound}
src := hostOOMSource(func(context.Context) (agentapi.GuestOOMKillsState, error) {
calls++
if answer != nil {
return agentapi.GuestOOMKillsState{}, answer
}
return agentapi.GuestOOMKillsState{Available: true, Containers: map[string]int64{"id": 2}}, nil
}, log.New(io.Discard, "", 0), func() time.Time { return clock })
for i := 0; i < 3; i++ {
if got := src(); got != nil {
t.Fatalf("an old agent: %v", got)
}
clock = clock.Add(30 * time.Second)
}
if calls != 1 {
t.Fatalf("an old agent was asked %d times in 90 s, want 1", calls)
}
clock = clock.Add(hostOOMRetryAfter)
answer = nil
if got := src(); got["id"] != 2 || calls != 2 {
t.Fatalf("after the back-off, an upgraded agent: %v (calls %d)", got, calls)
}
answer = errors.New("timeout")
if got := src(); got != nil || calls != 3 {
t.Fatalf("a transient failure: %v", got)
}
answer = nil
if got := src(); got["id"] != 2 || calls != 4 {
t.Fatalf("the next scan after a transient failure must ask again: %v (calls %d)", got, calls)
}
}
// The source is wired into the stack manager at startup (the seam-built-but-never-wired class).
// RED-PROOF: delete the SetHostOOMKills line in main.go → FAILS.
func TestR528_HostOOMSourceIsWiredAtStartup(t *testing.T) {
lines, _, _ := slice4CallLines(t)
if len(lines["SetHostOOMKills"]) == 0 || len(lines["hostOOMSource"]) == 0 {
t.Fatal("the host OOM counter is never wired — the 30-s scan would read Docker's flag only (R-528)")
}
}