admin
c885b5acf4
Measured on demo-felhom (N100, 16 GB): 61 apps deployed and removed through the product, sampled every 5 s; heavy apps with one real action (Immich 200 photos, Jellyfin transcode, Paperless OCR, Nextcloud 500 MB, ...). sizing.json for the estimator (tools/build_sizing.py), SIZING.md with the proposed minimum machine, METHOD.md. Power is RAPL CPU-package only (no meter). R-993 asks the operator to approve the minimum machine; R-994 lists catalog limits/requests that are too low; R-995 records leftover unnamed volumes after removal. R-231's dated check measured: dooplex-backup.service success, journal green.
201 lines
9.1 KiB
Python
201 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""build_sizing.py — build ../sizing.json (the estimator's data file) and ../evidence/apps-table.json.
|
|
|
|
Rebuild when the catalog changes: python3 -I tools/build_sizing.py [--catalog PATH]
|
|
|
|
Sources, per app, in the spec's order of preference (sizing-2026-10-13 TASK §7 B):
|
|
1 telemetry — the hub's App Telemetry, >= 7 days of 15-min reports on one box
|
|
(evidence/telemetry-30d.json, read from a hub DB copy 2026-10-13)
|
|
2 onboarding — a measured peak recorded by an earlier onboarding/bench session
|
|
(evidence/onboarding-peaks.json, collected 2026-10-13; per run, containers summed)
|
|
3 measured — this run's bench on the N100 (evidence/<batch>/summary.json)
|
|
4 unmeasured — the catalog's own mem_request / mem_limit
|
|
Rules: telemetry under 7 days counts only when nothing better exists (`short-telemetry`). When two
|
|
sources' peaks differ by more than 50 %, both are reported and the HIGHER is used for mem_peak_mb.
|
|
mem_typical_mb is the chosen source's typical figure. CPU always comes from the N100 bench when it
|
|
exists (the spec's unit: % of the whole N100), else from telemetry (docker % / 4, approximate).
|
|
"""
|
|
import glob, json, os, re, sys
|
|
from datetime import date
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
AUDIT = os.path.dirname(HERE)
|
|
EV = os.path.join(AUDIT, "evidence")
|
|
CATALOG = "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/templates"
|
|
if "--catalog" in sys.argv:
|
|
CATALOG = sys.argv[sys.argv.index("--catalog") + 1]
|
|
|
|
# CPU classes — stated once here and in METHOD.md
|
|
# light rest < 1 % of the whole N100 and the busiest minute < 15 %
|
|
# heavy-in-bursts the busiest minute >= 50 % (two of the four cores fully busy for a minute)
|
|
# medium everything between
|
|
def cpu_class(rest, peak1m):
|
|
if rest is None and peak1m is None:
|
|
return None
|
|
rest, peak1m = rest or 0, peak1m or 0
|
|
if peak1m >= 50:
|
|
return "heavy-in-bursts"
|
|
if rest < 1 and peak1m < 15:
|
|
return "light"
|
|
return "medium"
|
|
|
|
|
|
def mb(s):
|
|
if not s:
|
|
return None
|
|
m = re.match(r"\s*([\d.]+)\s*([GgMm])", str(s))
|
|
if not m:
|
|
return None
|
|
v = float(m.group(1))
|
|
return round(v * 1024 if m.group(2) in "Gg" else v)
|
|
|
|
|
|
def read_yaml_scalar(text, key):
|
|
m = re.search(rf"^\s*{key}:\s*\"?([^\"#\n]+?)\"?\s*(#.*)?$", text, re.M)
|
|
return m.group(1).strip() if m else None
|
|
|
|
|
|
def catalog():
|
|
apps = {}
|
|
for f in sorted(glob.glob(os.path.join(CATALOG, "*", ".felhom.yml"))):
|
|
slug = os.path.basename(os.path.dirname(f))
|
|
t = open(f, encoding="utf-8").read()
|
|
apps[slug] = dict(
|
|
display_name=read_yaml_scalar(t, "display_name") or read_yaml_scalar(t, "name") or slug,
|
|
mem_request=read_yaml_scalar(t, "mem_request"),
|
|
mem_limit=read_yaml_scalar(t, "mem_limit"),
|
|
lifecycle=read_yaml_scalar(t, "lifecycle") or "active",
|
|
)
|
|
return apps
|
|
|
|
|
|
def telemetry():
|
|
p = os.path.join(EV, "telemetry-30d.json")
|
|
out = {}
|
|
for r in json.load(open(p)) if os.path.exists(p) else []:
|
|
if r["app"] == "felhom-controller":
|
|
continue
|
|
cur = out.get(r["app"])
|
|
# the longest series wins (most days); ties -> more reports
|
|
if cur is None or (r["days"], r["reports"]) > (cur["days"], cur["reports"]):
|
|
out[r["app"]] = r
|
|
return out
|
|
|
|
|
|
def onboarding():
|
|
p = os.path.join(EV, "onboarding-peaks.json")
|
|
out = {}
|
|
for a in json.load(open(p)) if os.path.exists(p) else []:
|
|
runs = {}
|
|
for m in a.get("measured", []):
|
|
if m.get("what") != "peak" or m.get("basis") != "anon":
|
|
continue
|
|
run = m["cite"].split(":")[0]
|
|
runs.setdefault(run, {"mb": 0.0, "date": m.get("date"), "cite": m["cite"]})
|
|
runs[run]["mb"] += float(m["mb"])
|
|
typ = [m for m in a.get("measured", []) if m.get("what") == "typical"]
|
|
if runs:
|
|
best = max(runs.values(), key=lambda r: r["mb"])
|
|
out[a["slug"]] = dict(peak_mb=round(best["mb"]), date=best["date"], cite=best["cite"],
|
|
typical_mb=round(sum(float(m["mb"]) for m in typ)) if typ else None)
|
|
return out
|
|
|
|
|
|
def bench():
|
|
out = {}
|
|
for f in sorted(glob.glob(os.path.join(EV, "*", "summary.json"))):
|
|
tag = os.path.basename(os.path.dirname(f))
|
|
if tag == "floor":
|
|
continue
|
|
s = json.load(open(f))
|
|
evs = json.load(open(os.path.join(os.path.dirname(f), "events.json")))
|
|
deployed = {e["app"] for e in evs if e["kind"] == "deploy_done"}
|
|
day = date.fromtimestamp(min(e["ts"] for e in evs)).isoformat() if evs else None
|
|
for app, o in s["apps"].items():
|
|
if app in deployed and o.get("rest_samples"):
|
|
o = dict(o, batch=tag, date=day)
|
|
if app not in out or o["mem_peak_used_mb"] > out[app]["mem_peak_used_mb"]:
|
|
out[app] = o
|
|
return out
|
|
|
|
|
|
def floor():
|
|
p = os.path.join(EV, "floor", "summary.json")
|
|
if not os.path.exists(p):
|
|
return None
|
|
s = json.load(open(p))
|
|
plat = {k: v for k, v in s["apps"].items() if k in ("felhom-controller", "traefik", "cloudflared", "filebrowser")}
|
|
return dict(
|
|
host_mem_used_mb=s["host"]["host_mem_used_rest_avg_mb"],
|
|
guest_mem_anon_mb=s["host"]["guest_anon_rest_avg_mb"],
|
|
platform_containers_mb={k: v["mem_rest_avg_mb"] for k, v in plat.items()},
|
|
platform_containers_total_mb=sum(v["mem_rest_avg_mb"] for v in plat.values()),
|
|
note="host_mem_used_mb is MemTotal-MemAvailable on the N100 with only the platform running "
|
|
"(Proxmox, the agent, the guest and its four built-in containers); opengist, the one app "
|
|
"deployed there, is excluded from the platform figures and is ~23 MB",
|
|
)
|
|
|
|
|
|
def main():
|
|
cat, tel, onb, ben = catalog(), telemetry(), onboarding(), bench()
|
|
apps, table = [], []
|
|
for slug, c in cat.items():
|
|
t, o, b = tel.get(slug), onb.get(slug), ben.get(slug)
|
|
cands = [] # (name, typical, peak, date, box)
|
|
if t and t["days"] >= 7:
|
|
cands.append(("telemetry", t["mem_avg"], t["mem_p95"], t["last"], t["customer"]))
|
|
if o:
|
|
cands.append(("onboarding", o["typical_mb"], o["peak_mb"], o["date"], None))
|
|
if b:
|
|
cands.append(("measured", b["mem_rest_avg_mb"], b["mem_peak_used_mb"], b["date"], "demo-felhom (N100)"))
|
|
if t and t["days"] < 7:
|
|
cands.append(("short-telemetry", t["mem_avg"], t["mem_p95"], t["last"], t["customer"]))
|
|
if cands:
|
|
src, typ, peak, when, box = cands[0]
|
|
peaks = [x[2] for x in cands if x[2]]
|
|
disagree = bool(peaks) and max(peaks) > 1.5 * min(peaks)
|
|
if disagree:
|
|
peak = max(peaks)
|
|
if typ is None: # an onboarding peak with no typical figure: take the next source's
|
|
typ = next((x[1] for x in cands[1:] if x[1] is not None), None)
|
|
if typ is not None and peak is not None and peak < typ:
|
|
peak = typ # a peak below the typical figure is two sources' noise, never a real peak
|
|
else:
|
|
src, typ, peak, when, box, disagree = "unmeasured", mb(c["mem_request"]), mb(c["mem_limit"]), None, None, False
|
|
cpu_rest = b["cpu_rest_avg_pct"] if b else (round(t["cpu_avg_dockerpct"] / 4, 2) if t else None)
|
|
cpu_pk = b["cpu_peak_1m_pct"] if b else (round(t["cpu_max_15min"] / 4, 1) if t else None)
|
|
rec = dict(
|
|
slug=slug, display_name=c["display_name"],
|
|
mem_typical_mb=typ, mem_peak_mb=peak,
|
|
cpu_class=cpu_class(cpu_rest, cpu_pk), cpu_note=CPU_NOTES.get(slug, ""),
|
|
source=src, measured_on=box, date=when,
|
|
)
|
|
if c["lifecycle"] != "active":
|
|
rec["lifecycle"] = c["lifecycle"]
|
|
apps.append(rec)
|
|
table.append(dict(rec, catalog_request_mb=mb(c["mem_request"]), catalog_limit_mb=mb(c["mem_limit"]),
|
|
sources_disagree=disagree, cpu_rest_pct=cpu_rest, cpu_peak_1m_pct=cpu_pk,
|
|
telemetry=t, onboarding=o, bench=b))
|
|
power = json.load(open(os.path.join(EV, "power.json"))) if os.path.exists(os.path.join(EV, "power.json")) else None
|
|
out = dict(
|
|
schema=1, generated=date.today().isoformat(),
|
|
units=dict(mem="MB (MiB); docker-style used = memory.current - inactive_file", cpu="% of the whole Intel N100 (4 cores = 100 %)"),
|
|
cpu_classes=dict(light="rest < 1 % and busiest minute < 15 %", medium="between",
|
|
**{"heavy-in-bursts": "busiest minute >= 50 % (two cores busy for a minute)"}),
|
|
floor=floor(), power=power, apps=apps,
|
|
)
|
|
json.dump(out, open(os.path.join(AUDIT, "sizing.json"), "w"), indent=1, ensure_ascii=False)
|
|
json.dump(table, open(os.path.join(EV, "apps-table.json"), "w"), indent=1, ensure_ascii=False)
|
|
from collections import Counter
|
|
print("apps:", len(apps), dict(Counter(a["source"] for a in apps)))
|
|
|
|
|
|
# One line per app on what makes it burst — filled from the bench's action logs.
|
|
CPU_NOTES = {}
|
|
_notes = os.path.join(HERE, "cpu_notes.json")
|
|
if os.path.exists(_notes):
|
|
CPU_NOTES = json.load(open(_notes, encoding="utf-8"))
|
|
|
|
if __name__ == "__main__":
|
|
main()
|