admin/felhom.eu
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hub 0.178.0: the database speed (R-1346) — four indexes and one hint for the five statements that read whole tables (vouch fleet check, OS alarm agent versions, the 04:30 prune inside its write lock, the apps page); the manifest save in one transaction; the slow-statement line (SQL only, never values); evidence p1-r1344 + p2-r1346

Browse Source
This commit is contained in:
admin committed 2026-10-30 09:52:03 +01:00
1 parent be538975fc
commit a13c69b01a
15 files changed
+5412 -37
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

documentation/audits/release-2026-10-30/p1-r1344/health.csv
+4738
View File
File diff suppressed because it is too large. Load diff
documentation/audits/release-2026-10-30/p1-r1344/healthlog.py
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""healthlog.py <until-ISO-UTC> — R-1344 spike (Part 2, 2026-10-29): every 30 s, read each test box controller's Docker
health record (`docker inspect -f '{{json .State.Health}}' felhom-controller`, the probe Docker runs every 30 s:
`curl -f localhost:8080/api/health`, timeout 5 s) and append every NEW probe (keyed by its start time) to a CSV:
box, start, end, duration_ms, exit_code, status, failing_streak, container_started. Read-only; nothing on a box changes."""
import csv, json, os, subprocess, sys, time
from datetime import datetime, timezone
UNTIL = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00"))
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "health.csv")
BOXES = {"tester-1": ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "root@192.168.0.154"],
"demo-hp": ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "hp"]}
seen = {b: set() for b in BOXES}
new = not os.path.exists(OUT)
f = open(OUT, "a", newline="")
w = csv.writer(f)
if new:
w.writerow(["box", "start", "end", "duration_ms", "exit_code", "status", "failing_streak", "container_started", "read_at"])
def ts(s):
# Docker: 2026-10-29T11:11:37.145382734Z (nanoseconds) → microseconds for fromisoformat
s = s.rstrip("Z")
if "." in s:
a, b = s.split(".", 1)
s = a + "." + (b + "000000")[:6]
return datetime.fromisoformat(s).replace(tzinfo=timezone.utc)
while datetime.now(timezone.utc) < UNTIL:
for box, ssh in BOXES.items():
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
try:
r = subprocess.run(ssh + ["pct exec 9201 -- docker inspect -f '{{json .State.Health}}|{{.State.StartedAt}}' felhom-controller"],
capture_output=True, text=True, timeout=40)
line = [l for l in r.stdout.splitlines() if l.startswith("{")]
if not line:
w.writerow([box, "", "", "", "", "READ-FAILED", "", "", now]); f.flush(); continue
health, started = line[0].rsplit("|", 1)
h = json.loads(health)
for e in h.get("Log") or []:
if e["Start"] in seen[box]:
continue
seen[box].add(e["Start"])
d = (ts(e["End"]) - ts(e["Start"])).total_seconds() * 1000
w.writerow([box, e["Start"][:23], e["End"][:23], round(d), e["ExitCode"], h.get("Status"), h.get("FailingStreak"), started[:19], now])
f.flush()
except Exception as ex: # a read that fails is a row, never a silent gap
w.writerow([box, "", "", "", "", "READ-ERROR " + type(ex).__name__, "", "", now]); f.flush()
time.sleep(30)
documentation/audits/release-2026-10-30/p2-r1346/access-log-slow-requests.txt
+89
View File
@@ -0,0 +1,89 @@
# hub access log 2026-10-20 .. 2026-10-30 09:26 UTC: every request over 4 s except /api/v1/wait (long poll). Client addresses removed.
2026-10-20T09:32:16Z POST /configuration/artifacts 303 43537ms
2026-10-20T09:46:04Z GET /apps 200 10515ms
2026-10-20T12:11:01Z POST /configs/Tester4/edit 303 75400ms
2026-10-21T09:13:02Z POST /api/v1/host-enroll 201 11075ms
2026-10-21T12:03:50Z POST /api/v1/host-enroll 201 7973ms
2026-10-21T18:33:27Z POST /api/v1/host-enroll 201 11191ms
2026-10-22T02:30:00Z POST /api/v1/report 500 4911ms
2026-10-22T02:30:06Z POST /api/v1/report 500 4408ms
2026-10-22T02:30:08Z POST /api/v1/report 500 4507ms
2026-10-22T02:30:10Z POST /api/v1/report 500 4435ms
2026-10-22T02:30:16Z POST /api/v1/report 500 4787ms
2026-10-22T02:30:18Z POST /api/v1/report 500 4534ms
2026-10-22T02:30:19Z POST /api/v1/report 500 4921ms
2026-10-22T02:30:26Z POST /api/v1/report 500 4754ms
2026-10-22T02:30:27Z POST /api/v1/report 500 4711ms
2026-10-22T07:56:23Z POST /configuration/artifacts 303 37822ms
2026-10-22T09:41:17Z GET /apps 200 26162ms
2026-10-22T12:55:56Z POST /configs/Tester-2/reset 303 9013ms
2026-10-22T19:13:00Z POST /api/v1/host-enroll 201 8382ms
2026-10-23T07:58:04Z POST /configuration/artifacts 303 42136ms
2026-10-23T09:03:00Z GET /apps 200 16536ms
2026-10-23T12:32:59Z GET / 200 4524ms
2026-10-23T15:43:08Z GET /customers/demo-felhom 200 17970ms
2026-10-23T15:43:09Z GET /customers/demo-hp 200 33387ms
2026-10-23T15:59:35Z GET /customers/demo-hp 200 5317ms
2026-10-23T16:21:36Z GET /customers/demo-hp 200 6618ms
2026-10-23T16:34:14Z GET /customers/demo-hp 200 30839ms
2026-10-23T16:45:06Z GET /customers/demo-hp 200 4879ms
2026-10-23T16:57:39Z GET /customers/demo-hp 200 10595ms
2026-10-23T17:17:34Z GET /customers/demo-hp 200 20203ms
2026-10-23T17:28:17Z GET /customers/demo-hp 200 72876ms
2026-10-23T17:29:17Z GET /customers/demo-hp 200 12882ms
2026-10-23T17:30:31Z GET /customers/demo-hp 200 47569ms
2026-10-23T18:02:27Z POST /api/v1/host-report 200 7798ms
2026-10-23T18:02:24Z GET /customers/demo-hp 200 60175ms
2026-10-23T18:03:24Z GET /customers/demo-hp 200 4785ms
2026-10-23T18:48:37Z GET /customers/demo-hp 200 4428ms
2026-10-23T19:10:34Z GET /customers/demo-hp 200 5073ms
2026-10-24T00:02:54Z POST /api/v1/host-report 200 6025ms
2026-10-24T02:30:28Z POST /api/v1/report 500 4881ms
2026-10-24T02:30:38Z POST /api/v1/report 200 4914ms
2026-10-24T05:33:00Z GET /customers/demo-hp 200 21436ms
2026-10-24T06:02:27Z POST /api/v1/host-report 200 8684ms
2026-10-24T06:32:27Z POST /api/v1/host-report 200 5699ms
2026-10-24T07:44:29Z POST /configuration/artifacts 303 28888ms
2026-10-24T08:29:29Z GET /configuration 200 4122ms
2026-10-24T12:02:43Z POST /api/v1/host-report 200 4484ms
2026-10-24T12:43:13Z POST /api/v1/host-enroll 201 11177ms
2026-10-25T00:32:43Z POST /api/v1/host-report 200 6303ms
2026-10-25T01:02:43Z POST /api/v1/host-report 200 4714ms during-db-snapshot
2026-10-25T07:00:51Z POST /api/v1/host-report 200 6442ms during-db-snapshot
2026-10-25T07:02:43Z POST /api/v1/host-report 200 6306ms during-db-snapshot
2026-10-25T13:02:43Z POST /api/v1/host-report 200 4561ms during-db-snapshot
2026-10-25T13:32:43Z POST /api/v1/host-report 200 5223ms
2026-10-25T16:02:43Z POST /api/v1/host-report 200 4921ms
2026-10-25T16:51:48Z POST /api/v1/host-enroll 201 8096ms
2026-10-25T17:38:09Z POST /configuration/artifacts 303 39987ms
2026-10-25T19:03:06Z POST /api/v1/host-report 200 4626ms during-db-snapshot
2026-10-25T19:24:57Z POST /api/v1/host-enroll 201 8201ms
2026-10-26T09:04:22Z POST /configuration/artifacts 303 33643ms
2026-10-26T09:18:23Z POST /api/v1/host-report 200 4010ms
2026-10-26T09:25:31Z POST /configuration/artifacts 303 52286ms
2026-10-26T09:30:41Z POST /api/v1/host-enroll 201 11416ms
2026-10-26T11:40:34Z POST /api/v1/host-report 200 6146ms
2026-10-26T11:51:49Z POST /api/v1/host-enroll 201 7871ms
2026-10-26T19:03:23Z POST /api/v1/host-report 200 6178ms during-db-snapshot
2026-10-26T19:03:51Z GET /system 200 5775ms during-db-snapshot
2026-10-26T20:00:22Z POST /api/v1/host-enroll 201 8343ms
2026-10-27T07:46:39Z GET /apps 200 25055ms
2026-10-27T13:01:41Z POST /api/v1/host-report 200 5775ms during-db-snapshot
2026-10-27T13:03:40Z POST /api/v1/host-report 200 5178ms during-db-snapshot
2026-10-27T14:07:05Z GET /apps/romm 200 28984ms
2026-10-27T15:43:01Z POST /configuration/artifacts 303 36396ms
2026-10-27T19:03:15Z GET /customers/tester-1 200 8853ms during-db-snapshot
2026-10-27T19:31:40Z POST /api/v1/host-report 200 4341ms
2026-10-27T19:42:14Z GET /customers/tester-1 200 7418ms
2026-10-28T07:20:13Z GET /apps 200 30182ms
2026-10-28T19:03:40Z POST /api/v1/host-report 200 7081ms during-db-snapshot
2026-10-29T01:03:39Z POST /api/v1/host-report 200 9871ms during-db-snapshot
2026-10-29T03:30:45Z POST /api/v1/report 500 4707ms
2026-10-29T07:04:01Z POST /api/v1/host-report 200 5959ms during-db-snapshot
2026-10-29T09:21:19Z POST /configuration/artifacts 303 36965ms
2026-10-29T10:49:16Z GET /customers/tester-1 200 7823ms
2026-10-29T13:04:27Z POST /api/v1/host-report 200 4281ms during-db-snapshot
2026-10-29T13:03:19Z POST /configuration/artifacts 303 170022ms during-db-snapshot
2026-10-29T13:06:12Z POST /api/v1/host-report 200 4677ms
2026-10-29T13:04:59Z POST /configuration/artifacts 303 170431ms during-db-snapshot
2026-10-29T17:57:10Z POST /configuration/artifacts 303 38255ms
documentation/audits/release-2026-10-30/p2-r1346/idx-build.txt
+4
View File
@@ -0,0 +1,4 @@
09:45:54.873451 PROBE exec 6041ms err=<nil>: CREATE INDEX IF NOT EXISTS idx_reports_received ON reports(received_at)
09:46:29.076951 PROBE exec 34203ms err=<nil>: CREATE INDEX IF NOT EXISTS idx_host_reports_received ON host_reports(received_at)
09:46:35.123934 PROBE exec 6046ms err=<nil>: CREATE INDEX IF NOT EXISTS idx_reports_controller_version ON reports(controller_version)
09:46:52.169596 PROBE exec 17045ms err=<nil>: CREATE INDEX IF NOT EXISTS idx_host_reports_agent_version ON host_reports(host_id, agent_version, received_at)
documentation/audits/release-2026-10-30/p2-r1346/pages-read.txt
+14
View File
@@ -0,0 +1,14 @@
# Pages (4 KiB) read per statement, cold cache (cache_size 2 MB), on a copy of hub.db (snapshot 2026-10-30 07:00Z,
# 572 MB) in tmpfs on DooPlex. Before = live schema; after = with the four R-1346 indexes / the INDEXED BY hint.
statement before after
NewestReportedControllerVersion (vouch) 35623 90
HostAgentVersionsSeen demo-hp (OS alarm) 38570 118
Prune: DELETE reports (in the write lock) 16538 309
Prune: DELETE host_reports (write lock) 13559 704
GetFleetAppSummary 7 d (apps page) 15404 1437
# Table sizes: host_reports 327 MB (19 928 rows), reports 139 MB (21 742 rows), app_telemetry 33 MB (123 115 rows).
# controller_version and agent_version sit AFTER report_json in their rows: reading them walks the JSON's overflow pages.
# VACUUM INTO (the 6-hourly snapshot) while 9 600 writes ran on the same copy: the slowest write 1.5 ms — the snapshot
# takes no write lock; its cost on the live hub is 3 min of disk load (570 MB read + 570 MB written on Longhorn).
# Hub pod cgroup 2026-10-30 09:3x: memory.current 268 013 568 of max 268 435 456 (anon 85 MB, file cache 177 MB);
# cpu.stat nr_throttled 45 419 of 85 460 periods.
documentation/audits/release-2026-10-30/p2-r1346/probe-after.txt
+40
View File
@@ -0,0 +1,40 @@
09:49:31.734919 PROBE open+migrate 133ms
09:49:32.101110 PROBE round 1 GET /apps 354ms
09:49:33.424375 [WARN] slow statement: 1281ms (over 200ms): SELECT customer_id, received_at, report_json, health_status, cpu_percent, memory_percent, container_total, container_running, backup_last_sn
09:49:34.451805 [WARN] slow statement: 801ms (over 200ms): SELECT app_name, memory_current_mb FROM app_telemetry WHERE customer_id = ? AND reported_at = ( SELECT MAX(reported_at) FROM app_telemetry at
09:49:34.667219 PROBE round 1 GET /customers/demo-hp 2566ms
09:49:34.944156 PROBE round 1 GET /customers/tester-1 276ms
09:49:35.456023 PROBE round 1 GET /customers/demo-felhom 511ms
09:49:35.456067 PROBE status 503
09:49:35.456070 PROBE round 1 GET /system 0ms
09:49:35.459787 PROBE round 1 GET /configuration 3ms
09:49:35.499965 PROBE round 1 GET /hosts 40ms
09:49:35.634127 PROBE round 1 GET /hosts/demo-hp-bb76ea 134ms
09:49:35.659077 PROBE round 1 GET / 24ms
09:49:35.659403 PROBE status 302
09:49:35.659410 PROBE round 1 GET /configs/demo-hp/edit 0ms
09:49:35.668200 PROBE round 1 NewestReportedControllerVersion 8ms
09:49:35.675024 PROBE round 1 HostAgentVersionsSeen demo-hp-bb76ea 6ms
09:49:35.677130 PROBE round 1 HostAgentVersionsSeen tester-1-d70be4 2ms
09:49:35.683713 PROBE round 1 HostAgentVersionsSeen demo-felhom-8363b5 6ms
09:49:35.978921 PROBE round 1 GetFleetAppSummary 7d 295ms
09:49:35.983219 PROBE round 1 SetArtifactManifest (same values) 4ms
09:49:36.277407 PROBE round 2 GET /apps 294ms
09:49:37.372406 [WARN] slow statement: 891ms (over 200ms): SELECT app_name, memory_current_mb FROM app_telemetry WHERE customer_id = ? AND reported_at = ( SELECT MAX(reported_at) FROM app_telemetry at
09:49:37.630653 PROBE round 2 GET /customers/demo-hp 1353ms
09:49:37.874249 PROBE round 2 GET /customers/tester-1 243ms
09:49:38.270422 PROBE round 2 GET /customers/demo-felhom 396ms
09:49:38.270457 PROBE status 503
09:49:38.270460 PROBE round 2 GET /system 0ms
09:49:38.271677 PROBE round 2 GET /configuration 1ms
09:49:38.284324 PROBE round 2 GET /hosts 12ms
09:49:38.381573 PROBE round 2 GET /hosts/demo-hp-bb76ea 97ms
09:49:38.445815 PROBE round 2 GET / 64ms
09:49:38.446021 PROBE status 302
09:49:38.446024 PROBE round 2 GET /configs/demo-hp/edit 0ms
09:49:38.450512 PROBE round 2 NewestReportedControllerVersion 4ms
09:49:38.455056 PROBE round 2 HostAgentVersionsSeen demo-hp-bb76ea 4ms
09:49:38.456489 PROBE round 2 HostAgentVersionsSeen tester-1-d70be4 1ms
09:49:38.461097 PROBE round 2 HostAgentVersionsSeen demo-felhom-8363b5 4ms
09:49:38.776043 PROBE round 2 GetFleetAppSummary 7d 314ms
09:49:38.780616 PROBE round 2 SetArtifactManifest (same values) 4ms
documentation/audits/release-2026-10-30/p2-r1346/probe-before.txt
+23
View File
@@ -0,0 +1,23 @@
09:32:28.144871 PROBE open+migrate 75ms
09:32:28.766883 [WARN] slow statement: 608ms (over 200ms): SELECT app_name, MAX(display_name) as display_name, COUNT(DISTINCT customer_id) as deployment_count, AVG(memory_avg_mb) as avg_memory_mb, MAX
09:32:28.975341 PROBE round 1 GET /apps 817ms
09:32:30.483608 [WARN] slow statement: 1421ms (over 200ms): SELECT customer_id, received_at, report_json, health_status, cpu_percent, memory_percent, container_total, container_running, backup_last_sn
09:32:30.867327 [WARN] slow statement: 288ms (over 200ms): SELECT app_name, MAX(display_name), AVG(memory_avg_mb), MAX(memory_peak_mb), MAX(catalog_limit), SUM(log_errors), SUM(log_warnings) FROM app_
09:32:31.574715 [WARN] slow statement: 707ms (over 200ms): SELECT app_name, memory_current_mb FROM app_telemetry WHERE customer_id = ? AND reported_at = ( SELECT MAX(reported_at) FROM app_telemetry at
09:32:31.835182 PROBE round 1 GET /customers/demo-hp 2859ms
09:32:32.244337 PROBE round 1 GET /customers/tester-1 409ms
09:32:32.850731 PROBE round 1 GET /customers/demo-felhom 606ms
09:32:32.850785 PROBE status 503
09:32:32.850790 PROBE round 1 GET /system 0ms
09:32:32.854546 PROBE round 1 GET /configuration 3ms
09:32:32.873188 PROBE round 1 GET /hosts 18ms
09:32:32.958115 PROBE round 1 GET /hosts/demo-hp-bb76ea 84ms
09:32:33.012363 PROBE round 1 GET / 54ms
09:32:33.012600 PROBE status 302
09:32:33.012604 PROBE round 1 GET /configs/demo-hp/edit 0ms
09:32:34.739534 [WARN] slow statement: 1726ms (over 200ms): SELECT DISTINCT controller_version FROM reports WHERE controller_version IS NOT NULL AND controller_version != ''
09:32:34.739561 PROBE round 1 NewestReportedControllerVersion 1726ms
09:38:16.145431 [WARN] slow statement: 341405ms (over 200ms): SELECT COALESCE(agent_version, ''), MIN(received_at) FROM host_reports WHERE host_id = ? GROUP BY agent_version
09:38:16.145497 PROBE round 1 HostAgentVersionsSeen demo-hp-bb76ea 341405ms
09:40:30.083495 [WARN] slow statement: 133937ms (over 200ms): SELECT COALESCE(agent_version, ''), MIN(received_at) FROM host_reports WHERE host_id = ? GROUP BY agent_version
09:40:30.083533 PROBE round 1 HostAgentVersionsSeen tester-1-d70be4 133938ms