#!/usr/bin/python3 """rec.py — freeze-test recorder on the Felhom box (fixes 2026-10-21, Part 4). Every 10 s one JSON line to /root/rec/rec.jsonl: host PSI (memory/io/cpu), meminfo (free/available/cached/swap), vmstat counters (pgmajfault, pswpin/out, pgpgin, workingset_refault_*), loadavg, disk read/write sectors, the guest LXC cgroup (memory.current/max, memory.events, memory.stat file/anon/refault, memory.pressure, swap.current), and every Docker container's memory.current / memory.events / memory.stat anon+file (by container name). Reads /proc and /sys only.""" import json, os, re, subprocess, time OUT = "/root/rec/rec.jsonl"; os.makedirs("/root/rec", exist_ok=True) CG = "/sys/fs/cgroup/lxc/9201" def rd(p): try: return open(p).read() except Exception: return "" def kv(p, keys=None): d = {} for l in rd(p).splitlines(): a = l.split() if len(a) >= 2 and (keys is None or a[0].rstrip(":") in keys): try: d[a[0].rstrip(":")] = int(a[1]) except ValueError: pass return d def psi(p): out = {} for l in rd(p).splitlines(): m = re.match(r"(some|full) avg10=([\d.]+) avg60=([\d.]+) avg300=([\d.]+) total=(\d+)", l) if m: out[m.group(1)] = [float(m.group(2)), float(m.group(3)), int(m.group(5))] return out names = {} def cname(scope): cid = scope.split("docker-")[1].split(".scope")[0] if cid not in names: try: n = subprocess.run(["pct", "exec", "9201", "--", "docker", "inspect", "-f", "{{.Name}}", cid], capture_output=True, text=True, timeout=10).stdout.strip().lstrip("/") except Exception: n = "" names[cid] = n or cid[:12] return names[cid] VM = ("pgmajfault", "pswpin", "pswpout", "pgpgin", "pgpgout", "workingset_refault_file", "workingset_refault_anon", "oom_kill", "allocstall_normal", "pgscan_direct") MS = ("anon", "file", "file_mapped", "shmem", "workingset_refault_file", "workingset_refault_anon", "pgmajfault") while True: t = time.time() s = {"t": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(t)), "load": rd("/proc/loadavg").split()[:4], "psi": {k: psi(f"/proc/pressure/{k}") for k in ("memory", "io", "cpu")}, "mem": kv("/proc/meminfo", ("MemFree", "MemAvailable", "Cached", "SwapFree", "SwapTotal", "Dirty", "Writeback", "Active(file)", "Inactive(file)", "AnonPages", "Shmem")), "vm": kv("/proc/vmstat", VM), "disk": {l.split()[2]: [int(x) for x in l.split()[3:11]] for l in rd("/proc/diskstats").splitlines() if l.split()[2] in ("sda", "sdb")}, "guest": {"cur": int(rd(CG + "/memory.current") or 0), "max": rd(CG + "/memory.max").strip(), "swap": rd(CG + "/memory.swap.current").strip(), "events": kv(CG + "/memory.events"), "stat": kv(CG + "/memory.stat", MS), "psi": psi(CG + "/memory.pressure"), "iopsi": psi(CG + "/io.pressure")}, "c": {}} base = CG + "/ns/system.slice" try: for e in os.listdir(base): if e.startswith("docker-") and e.endswith(".scope"): p = base + "/" + e s["c"][cname(e)] = {"cur": int(rd(p + "/memory.current") or 0), "max": rd(p + "/memory.max").strip(), "ev": kv(p + "/memory.events", ("high", "max", "oom", "oom_kill")), "st": kv(p + "/memory.stat", ("anon", "file", "workingset_refault_file", "pgmajfault"))} except Exception as ex: s["cerr"] = str(ex) with open(OUT, "a") as f: f.write(json.dumps(s, separators=(",", ":")) + "\n") time.sleep(max(0.5, 10 - (time.time() - t)))