67 lines
3.6 KiB
Python
67 lines
3.6 KiB
Python
#!/usr/bin/python3
|
|
"""rec.py — freeze-test recorder on the Felhom box (fixes 2026-10-21, Part 4). Every 10 s one JSON line to
|
|
/root/rec/rec.jsonl: host PSI (memory/io/cpu), meminfo (free/available/cached/swap), vmstat counters (pgmajfault,
|
|
pswpin/out, pgpgin, workingset_refault_*), loadavg, disk read/write sectors, the guest LXC cgroup (memory.current/max,
|
|
memory.events, memory.stat file/anon/refault, memory.pressure, swap.current), and every Docker container's
|
|
memory.current / memory.events / memory.stat anon+file (by container name). Reads /proc and /sys only."""
|
|
import json, os, re, subprocess, time
|
|
OUT = "/root/rec/rec.jsonl"; os.makedirs("/root/rec", exist_ok=True)
|
|
CG = "/sys/fs/cgroup/lxc/9201"
|
|
def rd(p):
|
|
try:
|
|
return open(p).read()
|
|
except Exception:
|
|
return ""
|
|
def kv(p, keys=None):
|
|
d = {}
|
|
for l in rd(p).splitlines():
|
|
a = l.split()
|
|
if len(a) >= 2 and (keys is None or a[0].rstrip(":") in keys):
|
|
try:
|
|
d[a[0].rstrip(":")] = int(a[1])
|
|
except ValueError:
|
|
pass
|
|
return d
|
|
def psi(p):
|
|
out = {}
|
|
for l in rd(p).splitlines():
|
|
m = re.match(r"(some|full) avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+) total=(\d+)", l)
|
|
if m:
|
|
out[m.group(1)] = [float(m.group(2)), float(m.group(3)), int(m.group(5))]
|
|
return out
|
|
names = {}
|
|
def cname(scope):
|
|
cid = scope.split("docker-")[1].split(".scope")[0]
|
|
if cid not in names:
|
|
try:
|
|
n = subprocess.run(["pct", "exec", "9201", "--", "docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True, timeout=10).stdout.strip().lstrip("/")
|
|
except Exception:
|
|
n = ""
|
|
names[cid] = n or cid[:12]
|
|
return names[cid]
|
|
VM = ("pgmajfault", "pswpin", "pswpout", "pgpgin", "pgpgout", "workingset_refault_file", "workingset_refault_anon", "oom_kill", "allocstall_normal", "pgscan_direct")
|
|
MS = ("anon", "file", "file_mapped", "shmem", "workingset_refault_file", "workingset_refault_anon", "pgmajfault")
|
|
while True:
|
|
t = time.time()
|
|
s = {"t": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t)),
|
|
"load": rd("/proc/loadavg").split()[:4],
|
|
"psi": {k: psi(f"/proc/pressure/{k}") for k in ("memory", "io", "cpu")},
|
|
"mem": kv("/proc/meminfo", ("MemFree", "MemAvailable", "Cached", "SwapFree", "SwapTotal", "Dirty", "Writeback", "Active(file)", "Inactive(file)", "AnonPages", "Shmem")),
|
|
"vm": kv("/proc/vmstat", VM),
|
|
"disk": {l.split()[2]: [int(x) for x in l.split()[3:11]] for l in rd("/proc/diskstats").splitlines() if l.split()[2] in ("sda", "sdb")},
|
|
"guest": {"cur": int(rd(CG + "/memory.current") or 0), "max": rd(CG + "/memory.max").strip(), "swap": rd(CG + "/memory.swap.current").strip(),
|
|
"events": kv(CG + "/memory.events"), "stat": kv(CG + "/memory.stat", MS), "psi": psi(CG + "/memory.pressure"), "iopsi": psi(CG + "/io.pressure")},
|
|
"c": {}}
|
|
base = CG + "/ns/system.slice"
|
|
try:
|
|
for e in os.listdir(base):
|
|
if e.startswith("docker-") and e.endswith(".scope"):
|
|
p = base + "/" + e
|
|
s["c"][cname(e)] = {"cur": int(rd(p + "/memory.current") or 0), "max": rd(p + "/memory.max").strip(),
|
|
"ev": kv(p + "/memory.events", ("high", "max", "oom", "oom_kill")), "st": kv(p + "/memory.stat", ("anon", "file", "workingset_refault_file", "pgmajfault"))}
|
|
except Exception as ex:
|
|
s["cerr"] = str(ex)
|
|
with open(OUT, "a") as f:
|
|
f.write(json.dumps(s, separators=(",", ":")) + "\n")
|
|
time.sleep(max(0.5, 10 - (time.time() - t)))
|