R-528: the 30-s OOM scan also reads the host's per-container kernel counter (agent GET /guest/oom-kills)
gates / gates (push) Successful in 32s
gates / gates (push) Successful in 32s
A count above 0 in a running container's current run fires the existing app_oom line (source cgroup/docker in the details), whatever Docker's OOMKilled says. Old agent: 404 back-off; no agent: Docker's flag only. Tests TestR528_* red first / red-proved. Commit only.
This commit is contained in:
1 parent
e33cffaee1
commit
eed23e7594
9 files changed
+297
-7
No files matched your search
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||||
)
|
||||
|
||||
// hostOOMSource (R-528) is the 30-s dead-app scan's host-side OOM counter: one agent call per scan (5 s cap). An agent
|
||||
// without the route (404) is not asked again for hostOOMRetryAfter, and said once; any other failure is a nil answer
|
||||
// for this scan only (Docker's flag still works). Pinned by TestR528_HostOOMSource*.
|
||||
const hostOOMRetryAfter = 10 * time.Minute
|
||||
|
||||
func hostOOMSource(ask func(context.Context) (agentapi.GuestOOMKillsState, error), logger *log.Logger, now func() time.Time) func() map[string]int64 {
|
||||
var mu sync.Mutex
|
||||
var until time.Time
|
||||
said := false
|
||||
return func() map[string]int64 {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if now().Before(until) {
|
||||
return nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
st, err := ask(ctx)
|
||||
if err != nil {
|
||||
var se *agentapi.StatusError
|
||||
if errors.As(err, &se) && se.Code == http.StatusNotFound {
|
||||
until = now().Add(hostOOMRetryAfter)
|
||||
if !said {
|
||||
logger.Printf("[INFO] [deadapp] the agent has no OOM counter route (R-528) — Docker's OOM flag only; asked again in %s", hostOOMRetryAfter)
|
||||
said = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
logger.Printf("[WARN] [deadapp] the host OOM counter could not be read: %v — Docker's OOM flag only for this scan", err)
|
||||
return nil
|
||||
}
|
||||
if !st.Available {
|
||||
return nil
|
||||
}
|
||||
return st.Containers
|
||||
}
|
||||
}
|
||||
@@ -1035,6 +1035,9 @@ func main() {
|
||||
logger.Printf("[WARN] [deadapp] agent client init failed (%v) — the boot grace cannot learn of a crash boot; 90 s", err)
|
||||
} else {
|
||||
crashProbe = crashboot.AgentProbe(ac.CrashGuard)
|
||||
// R-528: the 30-s scan also asks the host for the kernel's per-container OOM counter (Docker's own flag can
|
||||
// stay false inside an LXC guest). TestR528_HostOOMSourceIsWiredAtStartup.
|
||||
stackMgr.SetHostOOMKills(hostOOMSource(ac.GuestOOMKills, logger, time.Now))
|
||||
}
|
||||
}
|
||||
bootGrace := crashboot.New(startTime, crashProbe, logger)
|
||||
@@ -1094,8 +1097,8 @@ func main() {
|
||||
logger.Printf("[WARN] [deadapp] OOM scan failed: %v", oerr)
|
||||
} else {
|
||||
for _, o := range ooms {
|
||||
logger.Printf("[WARN] [deadapp] %s: container %s was OOM-killed (started %s; kills %d, limit %s, peak %s)", o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak)
|
||||
notifier.NotifyAppOOM(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak)
|
||||
logger.Printf("[WARN] [deadapp] %s: container %s was OOM-killed (started %s; kills %d, limit %s, peak %s; seen by %s)", o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak, o.Source)
|
||||
notifier.NotifyAppOOMFrom(o.Stack, o.Container, o.StartedAt, o.Kills, o.MemLimit, o.Peak, o.Source)
|
||||
}
|
||||
}
|
||||
// v0.269.0 (`09` §3 decision 28, R-667): a crash loop or an OOM storm is STOPPED by the box.
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.dooplex.hu/admin/felhom-controller/internal/agentapi"
|
||||
)
|
||||
|
||||
// R-528: the scan's host counter source. An old agent (404) is asked again only after hostOOMRetryAfter; a transient
|
||||
// failure is retried at the next scan; the counters pass through.
|
||||
// RED-PROOF: drop the 404 back-off → an old agent is asked at every 30-s scan (calls 3, not 1) → FAILS.
|
||||
func TestR528_HostOOMSourceBacksOffAnOldAgent(t *testing.T) {
|
||||
clock := time.Date(2026, 10, 28, 23, 0, 0, 0, time.UTC)
|
||||
calls := 0
|
||||
var answer error = &agentapi.StatusError{Path: "/guest/oom-kills", Code: http.StatusNotFound}
|
||||
src := hostOOMSource(func(context.Context) (agentapi.GuestOOMKillsState, error) {
|
||||
calls++
|
||||
if answer != nil {
|
||||
return agentapi.GuestOOMKillsState{}, answer
|
||||
}
|
||||
return agentapi.GuestOOMKillsState{Available: true, Containers: map[string]int64{"id": 2}}, nil
|
||||
}, log.New(io.Discard, "", 0), func() time.Time { return clock })
|
||||
for i := 0; i < 3; i++ {
|
||||
if got := src(); got != nil {
|
||||
t.Fatalf("an old agent: %v", got)
|
||||
}
|
||||
clock = clock.Add(30 * time.Second)
|
||||
}
|
||||
if calls != 1 {
|
||||
t.Fatalf("an old agent was asked %d times in 90 s, want 1", calls)
|
||||
}
|
||||
clock = clock.Add(hostOOMRetryAfter)
|
||||
answer = nil
|
||||
if got := src(); got["id"] != 2 || calls != 2 {
|
||||
t.Fatalf("after the back-off, an upgraded agent: %v (calls %d)", got, calls)
|
||||
}
|
||||
answer = errors.New("timeout")
|
||||
if got := src(); got != nil || calls != 3 {
|
||||
t.Fatalf("a transient failure: %v", got)
|
||||
}
|
||||
answer = nil
|
||||
if got := src(); got["id"] != 2 || calls != 4 {
|
||||
t.Fatalf("the next scan after a transient failure must ask again: %v (calls %d)", got, calls)
|
||||
}
|
||||
}
|
||||
|
||||
// The source is wired into the stack manager at startup (the seam-built-but-never-wired class).
|
||||
// RED-PROOF: delete the SetHostOOMKills line in main.go → FAILS.
|
||||
func TestR528_HostOOMSourceIsWiredAtStartup(t *testing.T) {
|
||||
lines, _, _ := slice4CallLines(t)
|
||||
if len(lines["SetHostOOMKills"]) == 0 || len(lines["hostOOMSource"]) == 0 {
|
||||
t.Fatal("the host OOM counter is never wired — the 30-s scan would read Docker's flag only (R-528)")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user