#!/usr/bin/env python3 # soakstats.py REC OUTSIDE — the test night's numbers, per window (UTC): load (the uploads) and night (the chain). import json, sys, datetime as dt rec, out = sys.argv[1], sys.argv[2] W = {"load 19:56–23:38": ("2026-10-22T19:56:00Z", "2026-10-22T23:38:00Z"), "night 23:38–05:00": ("2026-10-22T23:38:00Z", "2026-10-23T05:00:00Z")} def T(s): return dt.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ") R = [json.loads(l) for l in open(rec) if l.strip()] O = [] for l in open(out): p = l.split() if len(p) < 3: continue code = p[1].split("=")[1]; lat = float(p[2]) if code != "000" else None O.append((T(p[0]), code, lat)) for name, (a, b) in W.items(): a, b = T(a), T(b) rs = [r for r in R if a <= T(r["t"]) < b] os_ = [o for o in O if a <= o[0] < b] ok = [o for o in os_ if o[1] == "200"] gaps = [(ok[i][0] - ok[i-1][0]).total_seconds() for i in range(1, len(ok))] wg = max(gaps) if gaps else None wgat = ok[gaps.index(wg)+1][0].strftime("%H:%M:%S") if gaps else "" slow = max((o for o in ok), key=lambda o: o[2]) if ok else None hf = [(r["psi"]["memory"]["full"][0], r["t"]) for r in rs] gf = [((r["guest"].get("psi") or {}).get("full", [0])[0] if isinstance(r["guest"].get("psi"), dict) else 0, r["t"]) for r in rs] h60 = max((r["psi"]["memory"]["full"][1] for r in rs), default=0) g60 = max(((r["guest"].get("psi") or {}).get("full", [0, 0])[1] if isinstance(r["guest"].get("psi"), dict) else 0) for r in rs) if rs else 0 over = sum(1 for (v, _), (g, _) in zip(hf, gf) if v > 20 or g > 20) ev0 = rs[0]["guest"]["events"] if rs else {}; ev1 = rs[-1]["guest"]["events"] if rs else {} def d(k): return int(ev1.get(k, 0)) - int(ev0.get(k, 0)) ook = int(rs[-1]["vm"].get("oom_kill", 0)) - int(rs[0]["vm"].get("oom_kill", 0)) if rs else 0 sw = max((int(r["guest"]["swap"] or 0) for r in rs), default=0) // (1024*1024) mh = max(hf, default=(0, "")); mg = max(gf, default=(0, "")) print(f"== {name}: rec samples {len(rs)}, health probes {len(os_)} / failed {len(os_)-len(ok)}") print(f" worst health gap between answers {wg} s at {wgat}; slowest answer {slow[2] if slow else None} s at {slow[0].strftime('%H:%M:%S') if slow else ''}") print(f" worst memory full avg10 host {mh[0]} % ({mh[1]}) / guest {mg[0]} % ({mg[1]}); samples > 20 %: {over}; worst avg60 host {h60} / guest {g60}") print(f" host oom_kill +{ook}; guest memory.events: max +{d('max')}, oom +{d('oom')}, oom_kill +{d('oom_kill')}, high +{d('high')}; guest swap max {sw} MB") nf = [o for o in os_ if o[1] != "200"] for o in nf[:5]: print(" non-200:", o[0].strftime("%H:%M:%S"), o[1])