admin
ce2a10e995
gates / gates (push) Successful in 27s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
58 lines
3.7 KiB
Python
58 lines
3.7 KiB
Python
"""burst.py — checklist 5.2's burst on 9202 for Karakeep at its final definition: 10 real public pages saved at once (the
|
|
household's ordinary heavy act), every container sampled from birth (sampler.sh), then removed. Evidence: box/karakeep-final/."""
|
|
import json, os, sys, time
|
|
import walk as w
|
|
sys.path.insert(0, "/mnt/5_hdd/felhom.eu/git/app-catalog-felhom.eu/scripts")
|
|
import upgrade_fixtures_box as fx
|
|
app, sub = "karakeep", "bookmarks"
|
|
EVD = f"{w.EV}/box/karakeep-final"; os.makedirs(EVD, exist_ok=True)
|
|
log = open(f"{EVD}/burst.txt", "a", buffering=1)
|
|
def say(*a):
|
|
w.say(*a); log.write(" ".join(map(str, a)) + "\n")
|
|
w.login()
|
|
for i in range(12):
|
|
w.sync_rescan(); h = w.guest("cd /var/lib/docker/volumes/felhom-controller-data/_data/data/catalog-cache && git log --oneline -1").strip()
|
|
if h.startswith(os.environ["WANT"]): break
|
|
time.sleep(10)
|
|
say("box clone:", h)
|
|
S = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "sampler.sh")).read()
|
|
w.guest(f"cat > /root/b-s.sh <<'SEOF'\n{S}\nSEOF\nrm -f /root/b-mem.csv.stop; (NAMES='karakeep karakeep-chrome karakeep-meilisearch' OUT=/root/b-mem.csv DUR=1500 nohup bash /root/b-s.sh >/dev/null 2>&1 &)")
|
|
say("deploy:", w.deploy(app, sub), "| limits:", w.guest("for c in karakeep karakeep-chrome karakeep-meilisearch; do echo -n \"$c=$(docker inspect -f '{{.HostConfig.Memory}}' $c) \"; done").strip())
|
|
t = fx.Karakeep().seed(w, sub, say)
|
|
key = fx.Karakeep()._key(w, sub, t)
|
|
urls = ["https://en.wikipedia.org/wiki/Budapest", "https://en.wikipedia.org/wiki/Hungary", "https://en.wikipedia.org/wiki/Danube",
|
|
"https://hu.wikipedia.org/wiki/Magyarorsz%C3%A1g", "https://www.python.org/", "https://docs.python.org/3/library/functions.html",
|
|
"https://en.wikipedia.org/wiki/Linux", "https://developer.mozilla.org/en-US/docs/Web/JavaScript", "https://www.gnu.org/licenses/agpl-3.0.html",
|
|
"https://en.wikipedia.org/wiki/Calendar"]
|
|
ids = []
|
|
for u in urls:
|
|
rc, code, out = w.app_curl(sub, "/api/v1/bookmarks", "-H", f"Authorization: Bearer {key}", "-H", "Content-Type: application/json",
|
|
data=json.dumps({"type": "link", "url": u}), method="POST")
|
|
ids.append(json.loads(out).get("id") if code in ("200", "201") else None)
|
|
say("10 pages saved at", time.strftime("%H:%M:%S"), "->", sum(1 for i in ids if i), "accepted")
|
|
done = 0
|
|
for _ in range(60):
|
|
time.sleep(10)
|
|
done = 0
|
|
for i in ids:
|
|
rc, c, o = w.app_curl(sub, f"/api/v1/bookmarks/{i}", "-H", f"Authorization: Bearer {key}")
|
|
if o.startswith("{") and (json.loads(o).get("content") or {}).get("crawledAt"):
|
|
done += 1
|
|
if done == len(ids): break
|
|
say("crawled:", done, "of", len(ids))
|
|
time.sleep(60)
|
|
w.guest("touch /root/b-mem.csv.stop"); time.sleep(3)
|
|
csv = w.guest("cat /root/b-mem.csv").strip().splitlines()
|
|
open(f"{EVD}/mem-burst.csv", "w").write("epoch,name,id,anon,current,swap,oom_kill,limit,restarts\n" + "\n".join(csv) + "\n")
|
|
per = {}
|
|
for l in csv:
|
|
p = l.split(",")
|
|
if len(p) >= 9 and p[3].isdigit():
|
|
d = per.setdefault(p[1], [0, 0, p[7], 0]); d[0] = max(d[0], int(p[3])); d[1] = max(d[1], int(p[6]) if p[6].isdigit() else 0); d[3] = max(d[3], int(p[8]) if p[8].isdigit() else 0)
|
|
for n, d in per.items():
|
|
say(f"5.2 burst {n}: peak anon {d[0]/1048576:.0f} MiB = {100*d[0]/int(d[2]):.1f} % of {int(d[2])//1048576} MiB; oom_kill {d[1]}; restarts {d[3]}")
|
|
w.ctl("POST", f"/api/stacks/{app}/stop"); time.sleep(20)
|
|
code, d = w.ctl("POST", f"/api/stacks/{app}/remove", {"remove_hdd_data": True, "remove_backups": True})
|
|
say("removed:", code, "| left:", w.guest("docker ps -a --format '{{.Names}}' | grep -c karakeep; docker volume ls -q | grep -c karakeep").split())
|
|
w.guest("rm -f /root/b-s.sh /root/b-mem.csv /root/b-mem.csv.stop")
|