64 lines
5.7 KiB
Bash
64 lines
5.7 KiB
Bash
cat > /tmp/mc2.sh <<'SHEOF'
|
|
#!/bin/bash
|
|
# meili_collect.sh <container> [interval_s=10] [duration_s=420] — R-1338 memory collector, read-only.
|
|
#
|
|
# Runs INSIDE the guest that runs Docker (bench: `pct exec 9401 -- bash meili_collect.sh wanderer-search`; a box guest
|
|
# the same way through `pct push` + `pct exec`). Changes nothing: it only reads cgroup v2 files, /proc and `docker
|
|
# inspect/logs/exec cat`. Prints:
|
|
# lines starting with '#' — a header (guest RAM/swap/CPUs, what the CONTAINER sees in /proc/meminfo and nproc, its
|
|
# Meilisearch env keys MEILI_* with values of *KEY* redacted, its start-up log lines) and,
|
|
# at the end, the top anonymous mappings of every process in the container; the full
|
|
# memory.stat at the start and the end;
|
|
# CSV — one row per sample, every interval_s for duration_s. Byte values, no rounding.
|
|
# NOTE (measured 2026-10-28 night, bench): an allocator's MADV_FREE'd pages stay counted in `anon` (and RssAnon) but sit
|
|
# on the INACTIVE FILE LRU, so inactive_file can exceed file. pglazyfree/pglazyfreed (pages) show it; the kernel drops
|
|
# them without swap when the cgroup reaches memory.max (ev_max counts those reclaim passes).
|
|
# The cgroup is looked up again on every sample, so a recreated container is followed (its row says the new id).
|
|
set -u
|
|
C=${1:?usage: meili_collect.sh <container> [interval_s] [duration_s]}
|
|
IV=${2:-10}; DUR=${3:-420}
|
|
cg_of() { local id; id=$(docker inspect -f '{{.Id}}' "$C" 2>/dev/null) || return 1
|
|
for p in /sys/fs/cgroup/system.slice/docker-$id.scope /sys/fs/cgroup/docker/$id; do [ -d "$p" ] && { echo "$p"; return 0; }; done; return 1; }
|
|
st() { awk -v k="$2" '$1==k{print $2; f=1} END{if(!f) print ""}' "$1/memory.stat"; }
|
|
ev() { awk -v k="$2" '$1==k{print $2; f=1} END{if(!f) print ""}' "$1/memory.events"; }
|
|
rd() { [ -r "$1" ] && cat "$1" || echo ""; }
|
|
mi() { awk -v k="$1:" '$1==k{print $2*1024}' /proc/meminfo; }
|
|
# sum a /proc/<pid>/status field (kB -> bytes) over every process in the cgroup
|
|
ps_sum() { local s=0 v; for p in $(cat "$1/cgroup.procs" 2>/dev/null); do v=$(awk -v k="$2:" '$1==k{print $2}' /proc/$p/status 2>/dev/null); s=$((s + ${v:-0})); done; echo $((s*1024)); }
|
|
|
|
echo "# meili_collect $(date -u +%FT%TZ) container=$C interval=${IV}s duration=${DUR}s guest=$(hostname) kernel=$(uname -r)"
|
|
echo "# guest: nproc=$(nproc) MemTotal=$(mi MemTotal) MemAvailable=$(mi MemAvailable) SwapTotal=$(mi SwapTotal) SwapFree=$(mi SwapFree)"
|
|
echo "# container sees: nproc=$(docker exec "$C" nproc 2>/dev/null || echo ?) $(docker exec "$C" sh -c 'grep -E "^(MemTotal|MemAvailable|SwapTotal):" /proc/meminfo' 2>/dev/null | tr -s ' ' | tr '\n' ' ')"
|
|
echo "# container image: $(docker inspect -f '{{.Config.Image}}' "$C" 2>/dev/null) started=$(docker inspect -f '{{.State.StartedAt}}' "$C" 2>/dev/null)"
|
|
echo "# container env (MEILI_*; *KEY* values redacted): $(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$C" 2>/dev/null | grep '^MEILI_' | sed -E 's/^(MEILI_[A-Z_]*KEY[A-Z_]*)=.*/\1=<redacted>/' | tr '\n' ' ')"
|
|
cg=$(cg_of) && echo "# cgroup: $cg memory.max=$(rd $cg/memory.max) memory.swap.max=$(rd $cg/memory.swap.max) memory.high=$(rd $cg/memory.high)"
|
|
echo "# start-up log lines (first 60 lines of docker logs that name memory/indexing/thread/version/config):"
|
|
docker logs "$C" 2>&1 | head -200 | grep -iE 'memory|index|thread|version|config|env|commit|database|max' | head -60 | sed 's/^/# /'
|
|
|
|
echo "ts,elapsed_s,container_id,memory_current,memory_peak,memory_max,swap_current,swap_peak,anon,file,file_mapped,active_file,inactive_file,active_anon,inactive_anon,shmem,kernel,slab,sock,pgmajfault,anon_thp,pglazyfree_pages,pglazyfreed_pages,ev_low,ev_high,ev_max,ev_oom,ev_oom_kill,proc_VmRSS,proc_RssAnon,proc_RssFile,proc_RssShmem,proc_VmSwap,proc_VmHWM,guest_MemAvailable,guest_SwapFree"
|
|
t0=$(date +%s); end=$((t0 + DUR))
|
|
if cg=$(cg_of); then echo "# memory.stat at start: $(tr '\n' ' ' < $cg/memory.stat)"; fi
|
|
while :; do
|
|
now=$(date +%s)
|
|
if cg=$(cg_of); then
|
|
id=$(basename "$cg" | sed -E 's/^docker-//; s/\.scope$//' | cut -c1-12)
|
|
echo "$(date -u +%T),$((now-t0)),$id,$(rd $cg/memory.current),$(rd $cg/memory.peak),$(rd $cg/memory.max),$(rd $cg/memory.swap.current),$(rd $cg/memory.swap.peak),$(st $cg anon),$(st $cg file),$(st $cg file_mapped),$(st $cg active_file),$(st $cg inactive_file),$(st $cg active_anon),$(st $cg inactive_anon),$(st $cg shmem),$(st $cg kernel),$(st $cg slab),$(st $cg sock),$(st $cg pgmajfault),$(st $cg anon_thp),$(st $cg pglazyfree),$(st $cg pglazyfreed),$(ev $cg low),$(ev $cg high),$(ev $cg max),$(ev $cg oom),$(ev $cg oom_kill),$(ps_sum $cg VmRSS),$(ps_sum $cg RssAnon),$(ps_sum $cg RssFile),$(ps_sum $cg RssShmem),$(ps_sum $cg VmSwap),$(ps_sum $cg VmHWM),$(mi MemAvailable),$(mi SwapFree)"
|
|
else
|
|
echo "$(date -u +%T),$((now-t0)),NO-CGROUP,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,"
|
|
fi
|
|
[ "$now" -ge "$end" ] && break
|
|
sleep "$IV"
|
|
done
|
|
if cg=$(cg_of); then
|
|
echo "# memory.stat at end: $(tr '\n' ' ' < $cg/memory.stat)"
|
|
echo "# memory.events at end: $(tr '\n' ' ' < $cg/memory.events)"
|
|
echo "# top anonymous mappings per process at the end (kB: Rss Anonymous Swap; mapping):"
|
|
for p in $(cat "$cg/cgroup.procs"); do
|
|
echo "# pid $p $(tr '\0' ' ' < /proc/$p/cmdline 2>/dev/null | cut -c1-80)"
|
|
awk '/^[0-9a-f]+-[0-9a-f]+ /{m=$0} /^Rss:/{r=$2} /^Anonymous:/{a=$2} /^Swap:/{s=$2; if (a>1024) printf "# %8d %8d %8d %s\n", r, a, s, m}' /proc/$p/smaps 2>/dev/null | sort -k3 -nr | head -12
|
|
grep -E '^(Rss|Anonymous|Swap|Pss_Anon|Pss_File):' /proc/$p/smaps_rollup 2>/dev/null | tr -s ' ' | tr '\n' ' ' | sed 's/^/# rollup: /'; echo
|
|
done
|
|
fi
|
|
SHEOF
|
|
bash /tmp/mc2.sh wanderer-search 1 0 | grep -v "^# \[" | cut -c1-600; rm -f /tmp/mc2.sh
|