40 lines
2.6 KiB
Python
40 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
||
# soakstats.py REC OUTSIDE — the test night's numbers, per window (UTC): load (the uploads) and night (the chain).
|
||
import json, sys, datetime as dt
|
||
rec, out = sys.argv[1], sys.argv[2]
|
||
W = {"load 19:56–23:38": ("2026-10-22T19:56:00Z", "2026-10-22T23:38:00Z"),
|
||
"night 23:38–05:00": ("2026-10-22T23:38:00Z", "2026-10-23T05:00:00Z")}
|
||
def T(s): return dt.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ")
|
||
R = [json.loads(l) for l in open(rec) if l.strip()]
|
||
O = []
|
||
for l in open(out):
|
||
p = l.split()
|
||
if len(p) < 3: continue
|
||
code = p[1].split("=")[1]; lat = float(p[2]) if code != "000" else None
|
||
O.append((T(p[0]), code, lat))
|
||
for name, (a, b) in W.items():
|
||
a, b = T(a), T(b)
|
||
rs = [r for r in R if a <= T(r["t"]) < b]
|
||
os_ = [o for o in O if a <= o[0] < b]
|
||
ok = [o for o in os_ if o[1] == "200"]
|
||
gaps = [(ok[i][0] - ok[i-1][0]).total_seconds() for i in range(1, len(ok))]
|
||
wg = max(gaps) if gaps else None
|
||
wgat = ok[gaps.index(wg)+1][0].strftime("%H:%M:%S") if gaps else ""
|
||
slow = max((o for o in ok), key=lambda o: o[2]) if ok else None
|
||
hf = [(r["psi"]["memory"]["full"][0], r["t"]) for r in rs]
|
||
gf = [((r["guest"].get("psi") or {}).get("full", [0])[0] if isinstance(r["guest"].get("psi"), dict) else 0, r["t"]) for r in rs]
|
||
h60 = max((r["psi"]["memory"]["full"][1] for r in rs), default=0)
|
||
g60 = max(((r["guest"].get("psi") or {}).get("full", [0, 0])[1] if isinstance(r["guest"].get("psi"), dict) else 0) for r in rs) if rs else 0
|
||
over = sum(1 for (v, _), (g, _) in zip(hf, gf) if v > 20 or g > 20)
|
||
ev0 = rs[0]["guest"]["events"] if rs else {}; ev1 = rs[-1]["guest"]["events"] if rs else {}
|
||
def d(k): return int(ev1.get(k, 0)) - int(ev0.get(k, 0))
|
||
ook = int(rs[-1]["vm"].get("oom_kill", 0)) - int(rs[0]["vm"].get("oom_kill", 0)) if rs else 0
|
||
sw = max((int(r["guest"]["swap"] or 0) for r in rs), default=0) // (1024*1024)
|
||
mh = max(hf, default=(0, "")); mg = max(gf, default=(0, ""))
|
||
print(f"== {name}: rec samples {len(rs)}, health probes {len(os_)} / failed {len(os_)-len(ok)}")
|
||
print(f" worst health gap between answers {wg} s at {wgat}; slowest answer {slow[2] if slow else None} s at {slow[0].strftime('%H:%M:%S') if slow else ''}")
|
||
print(f" worst memory full avg10 host {mh[0]} % ({mh[1]}) / guest {mg[0]} % ({mg[1]}); samples > 20 %: {over}; worst avg60 host {h60} / guest {g60}")
|
||
print(f" host oom_kill +{ook}; guest memory.events: max +{d('max')}, oom +{d('oom')}, oom_kill +{d('oom_kill')}, high +{d('high')}; guest swap max {sw} MB")
|
||
nf = [o for o in os_ if o[1] != "200"]
|
||
for o in nf[:5]: print(" non-200:", o[0].strftime("%H:%M:%S"), o[1])
|