admin
c885b5acf4
Measured on demo-felhom (N100, 16 GB): 61 apps deployed and removed through the product, sampled every 5 s; heavy apps with one real action (Immich 200 photos, Jellyfin transcode, Paperless OCR, Nextcloud 500 MB, ...). sizing.json for the estimator (tools/build_sizing.py), SIZING.md with the proposed minimum machine, METHOD.md. Power is RAPL CPU-package only (no meter). R-993 asks the operator to approve the minimum machine; R-994 lists catalog limits/requests that are too low; R-995 records leftover unnamed volumes after removal. R-231's dated check measured: dooplex-backup.service success, journal green.
187 lines
8.7 KiB
Python
187 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""analyze.py — turn one bench run's evidence into per-app numbers.
|
|
|
|
analyze.py evidence/<TAG> [--all-window]
|
|
|
|
Reads guest.tsv (5 s cgroup samples), containers.tsv (container -> compose project), events.json,
|
|
memory-peak.tsv and host.tsv. Writes <TAG>/summary.json and prints a table.
|
|
|
|
Definitions (stated once, used everywhere — METHOD.md repeats them):
|
|
- used = memory.current - inactive_file (what `docker stats` shows; what the fleet telemetry carries)
|
|
- anon = anonymous memory (the app's own heap; never reclaimable without swap)
|
|
- an app's value at a sample = the SUM over its containers at that timestamp
|
|
- rest window = events rest_start..rest_end (the last 15 min, nothing else being deployed)
|
|
- work window = the app's deploy_start .. rest_end (first start, any action, rest)
|
|
- CPU = d(usage_usec)/dt per container, summed per app, as % of the WHOLE N100 (4 cores = 100 %)
|
|
- cpu_peak_1m = the highest 60 s moving average in the work window
|
|
- host power = d(RAPL package energy)/dt — CPU package only, not wall power
|
|
"""
|
|
import json, os, sys, statistics as st, collections
|
|
|
|
NCPU = 4
|
|
|
|
|
|
def load_tsv(p):
|
|
rows = []
|
|
with open(p) as f:
|
|
hdr = f.readline().rstrip("\n").split("\t")
|
|
for line in f:
|
|
parts = line.rstrip("\n").split("\t")
|
|
if len(parts) == len(hdr):
|
|
rows.append(dict(zip(hdr, parts)))
|
|
return rows
|
|
|
|
|
|
def pct(xs, q):
|
|
xs = sorted(xs)
|
|
return xs[min(len(xs) - 1, int(q * len(xs)))] if xs else 0
|
|
|
|
|
|
def movavg_max(series, width):
|
|
"""series: sorted [(ts, value)]; highest mean over any window of `width` seconds."""
|
|
best, j, s = 0.0, 0, 0.0
|
|
for i in range(len(series)):
|
|
s += series[i][1]
|
|
while series[i][0] - series[j][0] >= width:
|
|
s -= series[j][1]
|
|
j += 1
|
|
n = i - j + 1
|
|
if series[i][0] - series[0][0] >= width - 5 and n:
|
|
best = max(best, s / n)
|
|
if not best and series:
|
|
best = sum(v for _, v in series) / len(series)
|
|
return best
|
|
|
|
|
|
def analyze(d):
|
|
g = load_tsv(os.path.join(d, "guest.tsv"))
|
|
proj = {}
|
|
cp = os.path.join(d, "containers.tsv")
|
|
if os.path.exists(cp):
|
|
for line in open(cp):
|
|
p = line.rstrip("\n").split("\t")
|
|
if len(p) >= 2:
|
|
proj[p[0]] = p[1] or p[0]
|
|
ev = json.load(open(os.path.join(d, "events.json"))) if os.path.exists(os.path.join(d, "events.json")) else []
|
|
t_rest0 = next((e["ts"] for e in ev if e["kind"] == "rest_start"), None)
|
|
t_rest1 = next((e["ts"] for e in ev if e["kind"] == "rest_end"), None)
|
|
starts = {e["app"]: e["ts"] for e in ev if e["kind"] == "deploy_start"}
|
|
|
|
# per container time series
|
|
mem = collections.defaultdict(dict) # app -> ts -> [used, anon]
|
|
cpu = collections.defaultdict(lambda: collections.defaultdict(float)) # app -> ts -> cores
|
|
last = {}
|
|
cpk = collections.defaultdict(lambda: [0, 0]) # container -> [peak used, peak anon]
|
|
for r in g:
|
|
ts = int(r["ts"]); name = r["name"]; app = proj.get(name, name)
|
|
used = int(r["mem_current"]) - int(r["inactive_file"] or 0)
|
|
anon = int(r["anon"] or 0)
|
|
m = mem[app].setdefault(ts, [0, 0]); m[0] += used; m[1] += anon
|
|
cpk[name][0] = max(cpk[name][0], used); cpk[name][1] = max(cpk[name][1], anon)
|
|
u = int(r["cpu_usage_usec"] or 0)
|
|
if name in last and ts > last[name][0] and u >= last[name][1]:
|
|
cpu[app][ts] += (u - last[name][1]) / 1e6 / (ts - last[name][0])
|
|
last[name] = (ts, u)
|
|
|
|
peaks = {}
|
|
climit = {}
|
|
pp = os.path.join(d, "memory-peak.tsv")
|
|
if os.path.exists(pp):
|
|
for r in load_tsv(pp):
|
|
mx = r.get("memory_max", "")
|
|
climit[r["name"]] = int(mx) if mx.isdigit() else None
|
|
app = proj.get(r["name"], r["name"])
|
|
try:
|
|
peaks.setdefault(app, [0, 0])
|
|
peaks[app][0] += int(r["memory_peak_bytes"] or 0)
|
|
peaks[app][1] += int(r["oom_kill"] or 0)
|
|
except ValueError:
|
|
pass
|
|
|
|
# Fallback for runs before memory_max was recorded: the applied compose, image -> next memory line
|
|
al = os.path.join(d, "applied-limits.txt")
|
|
if os.path.exists(al):
|
|
img_lim, cur = {}, None
|
|
for line in open(al):
|
|
if "image:" in line:
|
|
cur = line.split("image:", 1)[1].strip().split("@")[0]
|
|
elif cur and ("memory:" in line or "mem_limit:" in line):
|
|
v = line.split(":", 2)[-1].strip().strip('"')
|
|
unit = 1024 ** 3 if v[-1:] in "Gg" else 1024 ** 2
|
|
try:
|
|
img_lim[cur] = int(float(v[:-1]) * unit)
|
|
except ValueError:
|
|
pass
|
|
cur = None
|
|
for line in open(cp) if os.path.exists(cp) else []:
|
|
p = line.rstrip("\n").split("\t")
|
|
if len(p) >= 3 and not climit.get(p[0]) and p[2] in img_lim:
|
|
climit[p[0]] = img_lim[p[2]]
|
|
|
|
MB = 1024 * 1024
|
|
out = {}
|
|
for app, series in mem.items():
|
|
t0 = starts.get(app, min(series))
|
|
work = [(t, v) for t, v in sorted(series.items()) if t >= t0 and (t_rest1 is None or t <= t_rest1)]
|
|
rest = [(t, v) for t, v in work if t_rest0 and t >= t_rest0]
|
|
cw = sorted((t, c) for t, c in cpu[app].items() if t >= t0 and (t_rest1 is None or t <= t_rest1))
|
|
cr = [c for t, c in cw if t_rest0 and t >= t_rest0]
|
|
if not work:
|
|
continue
|
|
o = dict(
|
|
samples=len(work),
|
|
rest_samples=len(rest),
|
|
mem_rest_avg_mb=round(st.mean(v[0] for _, v in rest) / MB) if rest else None,
|
|
mem_rest_p95_mb=round(pct([v[0] for _, v in rest], .95) / MB) if rest else None,
|
|
mem_peak_used_mb=round(max(v[0] for _, v in work) / MB),
|
|
mem_peak_anon_mb=round(max(v[1] for _, v in work) / MB),
|
|
mem_rest_anon_avg_mb=round(st.mean(v[1] for _, v in rest) / MB) if rest else None,
|
|
kernel_memory_peak_mb=round(peaks[app][0] / MB) if app in peaks else None,
|
|
oom_kills=peaks[app][1] if app in peaks else None,
|
|
cpu_rest_avg_pct=round(100 * st.mean(cr) / NCPU, 2) if cr else None,
|
|
cpu_peak_1m_pct=round(100 * movavg_max(cw, 60) / NCPU, 1) if cw else None,
|
|
cpu_peak_5s_pct=round(100 * max((c for _, c in cw), default=0) / NCPU, 1),
|
|
work_seconds=work[-1][0] - work[0][0],
|
|
containers={n: dict(peak_used_mb=round(v[0] / MB), peak_anon_mb=round(v[1] / MB),
|
|
limit_mb=round(climit[n] / MB) if climit.get(n) else None)
|
|
for n, v in cpk.items() if proj.get(n, n) == app},
|
|
)
|
|
out[app] = o
|
|
|
|
# host side
|
|
h = load_tsv(os.path.join(d, "host.tsv")) if os.path.exists(os.path.join(d, "host.tsv")) else []
|
|
hs = {}
|
|
if len(h) > 2:
|
|
def watts(rows):
|
|
w = []
|
|
for a, b in zip(rows, rows[1:]):
|
|
dt = int(b["ts"]) - int(a["ts"]); de = int(b["rapl_pkg_uj"]) - int(a["rapl_pkg_uj"])
|
|
if dt > 0 and de >= 0:
|
|
w.append(de / 1e6 / dt)
|
|
return w
|
|
rest_h = [r for r in h if t_rest0 and int(r["ts"]) >= t_rest0 and (t_rest1 is None or int(r["ts"]) <= t_rest1)] or h
|
|
w_all, w_rest = watts(h), watts(rest_h)
|
|
hs = dict(
|
|
host_mem_used_rest_avg_mb=round(st.mean((int(r["mem_total_kb"]) - int(r["mem_available_kb"])) / 1024 for r in rest_h)),
|
|
host_mem_used_peak_mb=round(max((int(r["mem_total_kb"]) - int(r["mem_available_kb"])) / 1024 for r in h)),
|
|
guest_mem_current_rest_avg_mb=round(st.mean(int(r["guest_mem_current"]) for r in rest_h) / MB),
|
|
guest_anon_rest_avg_mb=round(st.mean(int(r["guest_anon"]) for r in rest_h) / MB),
|
|
rapl_pkg_w_rest_avg=round(st.mean(w_rest), 2) if w_rest else None,
|
|
rapl_pkg_w_peak_5s=round(max(w_all), 2) if w_all else None,
|
|
rapl_pkg_w_peak_1m=round(movavg_max([(int(b["ts"]), w) for b, w in zip(h[1:], w_all)], 60), 2) if w_all else None,
|
|
pkg_temp_c_max=round(max(int(r["pkg_temp_mC"]) for r in h) / 1000, 1),
|
|
)
|
|
res = dict(apps=out, host=hs)
|
|
json.dump(res, open(os.path.join(d, "summary.json"), "w"), indent=1)
|
|
return res
|
|
|
|
|
|
if __name__ == "__main__":
|
|
res = analyze(sys.argv[1])
|
|
print(f"{'app':22} {'rest':>6} {'p95':>6} {'peak':>6} {'anonpk':>6} {'kpeak':>6} {'cpuR%':>6} {'cpu1m%':>7} {'oom':>4}")
|
|
for a, o in sorted(res["apps"].items()):
|
|
print(f"{a:22} {str(o['mem_rest_avg_mb']):>6} {str(o['mem_rest_p95_mb']):>6} {o['mem_peak_used_mb']:>6} "
|
|
f"{o['mem_peak_anon_mb']:>6} {str(o['kernel_memory_peak_mb']):>6} {str(o['cpu_rest_avg_pct']):>6} "
|
|
f"{str(o['cpu_peak_1m_pct']):>7} {str(o['oom_kills']):>4}")
|
|
print(json.dumps(res["host"]))
|