admin
ac6c05bd7b
I1-I11 are CORRECTNESS invariants: they answer 'is the system telling the truth this cycle'. All 586 of them passed in run 1 while nothing at all watched whether disk usage, snapshot count, log volume, fd count or RSS climbs. Accumulation is exactly what depth was for, and it was missing from the invariant list. Adds c10growth.py (Campaign 2's controller_rss.tsv precedent, widened to 19 metrics) sampling every 90s as a SEPARATE process, so the in-flight run 2 did not have to be restarted. Attributes every sample to a cycle by reading the runner's status.txt, and records NA rather than dying when the box is down during a hard-reset or reboot atom. c10growth_report.py turns it into Campaign 2's table shape (start/end/min/max/ slope-per-cycle) and splits verdicts by class: growth in RSS/fd/volumes/images/ restarts is a LEAK; growth in backup storage or the qcow2 is expected accumulation, reported with a projection to cycle 45. Caught a bug in the sampler itself on the first analysis: MENTES_USED_MB appeared to jump 623 -> 5667 MB, which is exactly ROOT_USED_MB — when a drive is detached, /mnt/<name> reverts to a plain directory on root and df silently reports the ROOT filesystem. The same class of error as the agent's exactMount check, in the measurement code. Gated on mountpoint and red-proofed both ways: a real mount returns a number, a non-mount returns NA.
93 lines
3.6 KiB
Python
93 lines
3.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Campaign 10 — turn growth.tsv into a verdict per metric (Campaign 2's table shape).
|
|
|
|
The question is NOT "did it grow" — a backup soak SHOULD grow backup storage. It is
|
|
"does anything grow WITHOUT BOUND, per cycle, in a way 45 cycles would not survive".
|
|
So each metric gets: start, end, min, max, per-cycle slope (least squares over cycle
|
|
number), and a verdict that distinguishes expected accumulation from a leak.
|
|
"""
|
|
import collections, os, sys
|
|
|
|
STATE = "/mnt/5_hdd/felhom.eu/git/felhom.eu/documentation/tests/campaign10-evidence-2026-07-31/state"
|
|
SRC = os.path.join(STATE, "growth.tsv")
|
|
|
|
# metrics where monotonic growth is a LEAK, vs where it is expected accumulation
|
|
LEAK_IF_GROWING = {
|
|
"RSS_CTRL": "controller memory", "RSS_AGENT_KB": "agent memory",
|
|
"FD_CTRL": "controller file descriptors", "FD_AGENT": "agent file descriptors",
|
|
"N_VOLUMES": "docker volumes (redeploys must not orphan)",
|
|
"N_IMAGES": "docker images", "N_CONTAINERS": "running containers",
|
|
"CTRL_RESTARTS": "controller restart count",
|
|
}
|
|
EXPECTED_GROWTH = {
|
|
"BACKUPS_MB": "recovery units", "MENTES_USED_MB": "backup target",
|
|
"VZDUMP_N": "whole-guest dumps", "VMDISK_MB": "qcow2 on the host",
|
|
"LOGS_MB": "docker logs", "JOURNAL_MB": "guest journal",
|
|
"AGENT_JOURNAL_MB": "agent journal", "ROOT_USED_MB": "guest root fs",
|
|
"ADATOK_USED_MB": "data drive", "PGVOL_MB": "postgres volume",
|
|
"NVME_AVAIL_MB": "host free space (falls)",
|
|
}
|
|
|
|
|
|
def tofloat(v):
|
|
v = (v or "").strip().rstrip("MG")
|
|
try:
|
|
return float(v)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def main():
|
|
series = collections.defaultdict(list) # metric -> [(cycle, value)]
|
|
if not os.path.exists(SRC):
|
|
print("no growth.tsv yet"); return
|
|
for line in open(SRC):
|
|
f = line.rstrip("\n").split("\t")
|
|
if len(f) != 5 or f[0] == "epoch":
|
|
continue
|
|
_, _, cy, metric, val = f
|
|
v = tofloat(val)
|
|
try:
|
|
c = int(cy)
|
|
except Exception:
|
|
continue
|
|
if v is not None:
|
|
series[metric].append((c, v))
|
|
|
|
print("%-18s %10s %10s %10s %10s %12s %s" %
|
|
("metric", "start", "end", "min", "max", "slope/cycle", "verdict"))
|
|
print("-" * 104)
|
|
concerns = []
|
|
for m in sorted(series):
|
|
pts = series[m]
|
|
if len(pts) < 3:
|
|
continue
|
|
xs = [p[0] for p in pts]; ys = [p[1] for p in pts]
|
|
n = len(xs); mx = sum(xs) / n; my = sum(ys) / n
|
|
den = sum((x - mx) ** 2 for x in xs)
|
|
slope = (sum((xs[i] - mx) * (ys[i] - my) for i in range(n)) / den) if den else 0.0
|
|
start, end, lo, hi = ys[0], ys[-1], min(ys), max(ys)
|
|
span = max(xs) - min(xs)
|
|
# projection to cycle 45 from the last observed point
|
|
proj = end + slope * max(0, 45 - max(xs))
|
|
if m in LEAK_IF_GROWING:
|
|
rel = (slope * max(span, 1)) / (abs(start) or 1)
|
|
verdict = "LEAK-SUSPECT" if (slope > 0 and rel > 0.25) else "stable — no leak"
|
|
if verdict.startswith("LEAK"):
|
|
concerns.append((m, slope, end, proj))
|
|
else:
|
|
label = EXPECTED_GROWTH.get(m, "")
|
|
verdict = "expected accumulation (%s); proj@c45=%.0f" % (label, proj)
|
|
print("%-18s %10.1f %10.1f %10.1f %10.1f %12.2f %s" % (m, start, end, lo, hi, slope, verdict))
|
|
print()
|
|
if concerns:
|
|
print("CONCERNS (growth where growth is a leak):")
|
|
for m, s, e, p in concerns:
|
|
print(" %s slope=%.2f/cycle now=%.1f projected@c45=%.1f" % (m, s, e, p))
|
|
else:
|
|
print("No leak-class metric grew materially.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|