Campaign 10: add monotonic-growth sampling — the half the invariants cannot see
I1-I11 are CORRECTNESS invariants: they answer 'is the system telling the truth this cycle'. All 586 of them passed in run 1 while nothing at all watched whether disk usage, snapshot count, log volume, fd count or RSS climbs. Accumulation is exactly what depth was for, and it was missing from the invariant list. Adds c10growth.py (Campaign 2's controller_rss.tsv precedent, widened to 19 metrics) sampling every 90s as a SEPARATE process, so the in-flight run 2 did not have to be restarted. Attributes every sample to a cycle by reading the runner's status.txt, and records NA rather than dying when the box is down during a hard-reset or reboot atom. c10growth_report.py turns it into Campaign 2's table shape (start/end/min/max/ slope-per-cycle) and splits verdicts by class: growth in RSS/fd/volumes/images/ restarts is a LEAK; growth in backup storage or the qcow2 is expected accumulation, reported with a projection to cycle 45. Caught a bug in the sampler itself on the first analysis: MENTES_USED_MB appeared to jump 623 -> 5667 MB, which is exactly ROOT_USED_MB — when a drive is detached, /mnt/<name> reverts to a plain directory on root and df silently reports the ROOT filesystem. The same class of error as the agent's exactMount check, in the measurement code. Gated on mountpoint and red-proofed both ways: a real mount returns a number, a non-mount returns NA.
This commit is contained in:
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Campaign 10 — monotonic-growth sampler (Campaign 2's controller_rss.tsv precedent, widened).
|
||||
|
||||
WHY THIS EXISTS, separately from the invariants: I1-I11 are CORRECTNESS checks — they answer
|
||||
"is the system telling the truth this cycle". Every one of them can pass 45 times in a row while
|
||||
disk usage, snapshot count, log volume or RSS climbs steadily. Accumulation is the half a
|
||||
targeted session structurally cannot test, and it is the whole point of depth.
|
||||
|
||||
Runs as its own process alongside c10run.py so the in-flight soak does not have to be restarted.
|
||||
Reads the runner's status.txt so every sample is attributed to a cycle. Tolerates the box being
|
||||
down (hard-reset / reboot atoms) by recording NA rather than dying.
|
||||
|
||||
One row per sample, long format: epoch, iso, cycle, metric, value.
|
||||
"""
|
||||
import os, re, subprocess, time
|
||||
|
||||
EV = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/tests/campaign10-evidence-2026-07-31"
|
||||
STATE = os.path.join(EV, "state")
|
||||
OUT = os.path.join(STATE, "growth.tsv")
|
||||
STATUS = os.path.join(STATE, "status.txt")
|
||||
INTERVAL = 90
|
||||
|
||||
HOME = os.path.expanduser("~/.config/campaign10")
|
||||
import json
|
||||
HOSTPW = json.load(open(os.path.join(HOME, "host-recovery.json")))["password"]
|
||||
|
||||
|
||||
def box(cmd, timeout=60):
|
||||
try:
|
||||
p = subprocess.run(["sshpass", "-e", "ssh", "-o", "StrictHostKeyChecking=no",
|
||||
"-o", "ConnectTimeout=12", "-J", "demo-hp", "root@192.168.0.105",
|
||||
"LC_ALL=C " + cmd],
|
||||
capture_output=True, text=True, timeout=timeout,
|
||||
env=dict(os.environ, SSHPASS=HOSTPW, LC_ALL="C"))
|
||||
return p.stdout.strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def hp(cmd, timeout=60):
|
||||
try:
|
||||
p = subprocess.run(["ssh", "-o", "ConnectTimeout=12", "demo-hp", "LC_ALL=C " + cmd],
|
||||
capture_output=True, text=True, timeout=timeout)
|
||||
return p.stdout.strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def cycle_now():
|
||||
try:
|
||||
m = re.search(r"cycle=(\d+)", open(STATUS).read())
|
||||
return m.group(1) if m else "?"
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
|
||||
def num(s, default="NA"):
|
||||
m = re.search(r"-?\d+(?:\.\d+)?", s or "")
|
||||
return m.group(0) if m else default
|
||||
|
||||
|
||||
def sample():
|
||||
"""One batched round-trip per host keeps the load off the box the soak is abusing."""
|
||||
rows = {}
|
||||
|
||||
# --- inside the guest: controller/app footprint, log volume, docker object counts ---
|
||||
g = box(
|
||||
"pct exec 9201 -- bash -c '"
|
||||
"echo RSS_CTRL=$(docker stats --no-stream --format \"{{.Name}} {{.MemUsage}}\" felhom-controller 2>/dev/null | awk \"{print \\$2}\"); "
|
||||
"echo FD_CTRL=$(docker inspect -f {{.State.Pid}} felhom-controller 2>/dev/null | xargs -I{} sh -c \"ls /proc/{}/fd 2>/dev/null | wc -l\"); "
|
||||
"echo CTRL_RESTARTS=$(docker inspect -f {{.RestartCount}} felhom-controller 2>/dev/null); "
|
||||
"echo N_CONTAINERS=$(docker ps -q 2>/dev/null | wc -l); "
|
||||
"echo N_VOLUMES=$(docker volume ls -q 2>/dev/null | wc -l); "
|
||||
"echo N_IMAGES=$(docker images -q 2>/dev/null | wc -l); "
|
||||
"echo LOGS_MB=$(du -sm /var/lib/docker/containers 2>/dev/null | cut -f1); "
|
||||
"echo JOURNAL_MB=$(journalctl --disk-usage 2>/dev/null | grep -oE \"[0-9.]+[MG]\" | head -1); "
|
||||
"echo BACKUPS_MB=$(du -sm /mnt/sys_drive/felhom-data/backups 2>/dev/null | cut -f1); "
|
||||
"echo PGVOL_MB=$(du -sm /var/lib/docker/volumes/rallly_rallly_postgres_data 2>/dev/null | cut -f1); "
|
||||
"'")
|
||||
for line in g.splitlines():
|
||||
if "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
rows[k.strip()] = num(v) if k.strip() != "JOURNAL_MB" else (v.strip() or "NA")
|
||||
|
||||
# --- on the PVE host (the VM): agent footprint + the drives themselves ---
|
||||
h = box(
|
||||
"bash -c '"
|
||||
"echo RSS_AGENT_KB=$(ps -o rss= -C felhom-agent 2>/dev/null | head -1); "
|
||||
"echo FD_AGENT=$(pgrep -x felhom-agent | head -1 | xargs -I{} sh -c \"ls /proc/{}/fd 2>/dev/null | wc -l\"); "
|
||||
"echo ROOT_USED_MB=$(df -BM --output=used / | tail -1 | tr -dc 0-9); "
|
||||
# MUST gate on mountpoint: when a drive is detached, /mnt/<name> reverts to a plain
|
||||
# directory on root and df silently reports the ROOT filesystem — which read as a
|
||||
# 623->5667 MB "jump" (exactly ROOT_USED_MB) in the first samples. Same class of error
|
||||
# as the agent's exactMount check, in the measurement code.
|
||||
"echo MENTES_USED_MB=$(mountpoint -q /mnt/mentes && df -BM --output=used /mnt/mentes | tail -1 | tr -dc 0-9 || echo NA); "
|
||||
"echo ADATOK_USED_MB=$(mountpoint -q /mnt/adatok && df -BM --output=used /mnt/adatok | tail -1 | tr -dc 0-9 || echo NA); "
|
||||
"echo VZDUMP_N=$(mountpoint -q /mnt/mentes && ls -1 /mnt/mentes/dump 2>/dev/null | wc -l || echo NA); "
|
||||
"echo AGENT_JOURNAL_MB=$(journalctl -u felhom-agent --disk-usage 2>/dev/null | grep -oE \"[0-9.]+[MG]\" | head -1); "
|
||||
"'")
|
||||
for line in h.splitlines():
|
||||
if "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
rows[k.strip()] = num(v) if k.strip() != "AGENT_JOURNAL_MB" else (v.strip() or "NA")
|
||||
|
||||
# --- on demo-hp: the qcow2 files actually growing on the host ---
|
||||
d = hp("du -sm /mnt/nvme-1tb/images/311 2>/dev/null | cut -f1; df -BM --output=avail /mnt/nvme-1tb | tail -1 | tr -dc 0-9")
|
||||
parts = [p for p in d.splitlines() if p.strip()]
|
||||
rows["VMDISK_MB"] = num(parts[0]) if len(parts) > 0 else "NA"
|
||||
rows["NVME_AVAIL_MB"] = num(parts[1]) if len(parts) > 1 else "NA"
|
||||
return rows
|
||||
|
||||
|
||||
def main():
|
||||
new = not os.path.exists(OUT)
|
||||
with open(OUT, "a") as fh:
|
||||
if new:
|
||||
fh.write("epoch\tiso\tcycle\tmetric\tvalue\n")
|
||||
while True:
|
||||
try:
|
||||
r = sample()
|
||||
ep = int(time.time())
|
||||
iso = time.strftime("%FT%TZ", time.gmtime())
|
||||
cy = cycle_now()
|
||||
with open(OUT, "a") as fh:
|
||||
for k in sorted(r):
|
||||
fh.write("%d\t%s\t%s\t%s\t%s\n" % (ep, iso, cy, k, r[k]))
|
||||
except Exception as e:
|
||||
with open(OUT, "a") as fh:
|
||||
fh.write("%d\t%s\t%s\tSAMPLER_ERROR\t%s\n"
|
||||
% (int(time.time()), time.strftime("%FT%TZ", time.gmtime()), cycle_now(), e))
|
||||
time.sleep(INTERVAL)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Campaign 10 — turn growth.tsv into a verdict per metric (Campaign 2's table shape).
|
||||
|
||||
The question is NOT "did it grow" — a backup soak SHOULD grow backup storage. It is
|
||||
"does anything grow WITHOUT BOUND, per cycle, in a way 45 cycles would not survive".
|
||||
So each metric gets: start, end, min, max, per-cycle slope (least squares over cycle
|
||||
number), and a verdict that distinguishes expected accumulation from a leak.
|
||||
"""
|
||||
import collections, os, sys
|
||||
|
||||
STATE = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/tests/campaign10-evidence-2026-07-31/state"
|
||||
SRC = os.path.join(STATE, "growth.tsv")
|
||||
|
||||
# metrics where monotonic growth is a LEAK, vs where it is expected accumulation
|
||||
LEAK_IF_GROWING = {
|
||||
"RSS_CTRL": "controller memory", "RSS_AGENT_KB": "agent memory",
|
||||
"FD_CTRL": "controller file descriptors", "FD_AGENT": "agent file descriptors",
|
||||
"N_VOLUMES": "docker volumes (redeploys must not orphan)",
|
||||
"N_IMAGES": "docker images", "N_CONTAINERS": "running containers",
|
||||
"CTRL_RESTARTS": "controller restart count",
|
||||
}
|
||||
EXPECTED_GROWTH = {
|
||||
"BACKUPS_MB": "recovery units", "MENTES_USED_MB": "backup target",
|
||||
"VZDUMP_N": "whole-guest dumps", "VMDISK_MB": "qcow2 on the host",
|
||||
"LOGS_MB": "docker logs", "JOURNAL_MB": "guest journal",
|
||||
"AGENT_JOURNAL_MB": "agent journal", "ROOT_USED_MB": "guest root fs",
|
||||
"ADATOK_USED_MB": "data drive", "PGVOL_MB": "postgres volume",
|
||||
"NVME_AVAIL_MB": "host free space (falls)",
|
||||
}
|
||||
|
||||
|
||||
def tofloat(v):
|
||||
v = (v or "").strip().rstrip("MG")
|
||||
try:
|
||||
return float(v)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
series = collections.defaultdict(list) # metric -> [(cycle, value)]
|
||||
if not os.path.exists(SRC):
|
||||
print("no growth.tsv yet"); return
|
||||
for line in open(SRC):
|
||||
f = line.rstrip("\n").split("\t")
|
||||
if len(f) != 5 or f[0] == "epoch":
|
||||
continue
|
||||
_, _, cy, metric, val = f
|
||||
v = tofloat(val)
|
||||
try:
|
||||
c = int(cy)
|
||||
except Exception:
|
||||
continue
|
||||
if v is not None:
|
||||
series[metric].append((c, v))
|
||||
|
||||
print("%-18s %10s %10s %10s %10s %12s %s" %
|
||||
("metric", "start", "end", "min", "max", "slope/cycle", "verdict"))
|
||||
print("-" * 104)
|
||||
concerns = []
|
||||
for m in sorted(series):
|
||||
pts = series[m]
|
||||
if len(pts) < 3:
|
||||
continue
|
||||
xs = [p[0] for p in pts]; ys = [p[1] for p in pts]
|
||||
n = len(xs); mx = sum(xs) / n; my = sum(ys) / n
|
||||
den = sum((x - mx) ** 2 for x in xs)
|
||||
slope = (sum((xs[i] - mx) * (ys[i] - my) for i in range(n)) / den) if den else 0.0
|
||||
start, end, lo, hi = ys[0], ys[-1], min(ys), max(ys)
|
||||
span = max(xs) - min(xs)
|
||||
# projection to cycle 45 from the last observed point
|
||||
proj = end + slope * max(0, 45 - max(xs))
|
||||
if m in LEAK_IF_GROWING:
|
||||
rel = (slope * max(span, 1)) / (abs(start) or 1)
|
||||
verdict = "LEAK-SUSPECT" if (slope > 0 and rel > 0.25) else "stable — no leak"
|
||||
if verdict.startswith("LEAK"):
|
||||
concerns.append((m, slope, end, proj))
|
||||
else:
|
||||
label = EXPECTED_GROWTH.get(m, "")
|
||||
verdict = "expected accumulation (%s); proj@c45=%.0f" % (label, proj)
|
||||
print("%-18s %10.1f %10.1f %10.1f %10.1f %12.2f %s" % (m, start, end, lo, hi, slope, verdict))
|
||||
print()
|
||||
if concerns:
|
||||
print("CONCERNS (growth where growth is a leak):")
|
||||
for m, s, e, p in concerns:
|
||||
print(" %s slope=%.2f/cycle now=%.1f projected@c45=%.1f" % (m, s, e, p))
|
||||
else:
|
||||
print("No leak-class metric grew materially.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user