admin
52b6599d97
gates / gates (push) Successful in 6m36s
Read-only. fetch.py (polite, 5 s apart, 34 pages, all 200), parse.py (own text only; four parser defects found by the controls and fixed), build.py + spotcheck.py (stdlib). tags.csv holds post numbers and tags only - no username, author id, salt or post text in git. FINDINGS.md (2,908 posts, 195 people, 82 beginner threads in 7 clusters), FELHOM.md (catalog gap, hardware fit, 15 website questions, objections, ten ranked recommendations), METHOD.md (controls, spot-check: role and hardware below 85 % after two rounds, the table-driving splits 92.5 % and 97.3 %), REPORT.md. R-990: operator rules on the recommendations. R-991: no buyer-readable hardware floor. Claude-Session: https://claude.ai/code/session_012qRErfCoiTkvDK9N5XHbzb
320 lines
16 KiB
Python
320 lines
16 KiB
Python
#!/usr/bin/env python3
|
|
"""Merge the helper tags, build threads, write tags.csv (committed) and print the FINDINGS tables.
|
|
|
|
Read-only research (forum-research-2026-10-12). Stdlib only.
|
|
Inputs (cache, never in git): posts.jsonl, tags/tags_*.csv, clusters.json (optional).
|
|
Output: tags.csv here (num + tags only: no text, no author_id, no gist) and tables on stdout.
|
|
|
|
Threads (spec 7 C): walk up reply_to from a post; the first ancestor-or-self whose role ends in
|
|
"-asks" opens a thread; keep walking while the next "-asks" ancestor is by the SAME person (a
|
|
follow-up question is the same thread). A post with no "-asks" ancestor belongs to no thread.
|
|
A thread is a beginner thread when its opening post is beginner-asks.
|
|
Thread resolved = yes if any post by the asker in it is tagged yes, else no if any is no, else unknown.
|
|
|
|
"People" = distinct author_id. Usage: python3 build.py [--write-tags]
|
|
"""
|
|
import collections
|
|
import csv
|
|
import glob
|
|
import json
|
|
import os
|
|
import statistics
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
CACHE = os.path.expanduser("~/research-cache/prohardver-hazi-szerver")
|
|
COLS = ["num", "year", "role", "beginner_evidence", "question_type", "hardware", "hardware_status",
|
|
"apps", "apps_status", "platform", "platform_status", "pain", "budget_huf", "resolved", "turnkey"]
|
|
|
|
# Spelling variants the taggers produced -> one canonical name (filled from the merged tag values).
|
|
APP_ALIASES = {
|
|
"qbittorrent": "qBittorrent", "qbit": "qBittorrent", "homeassistant": "Home Assistant",
|
|
"home assistant os": "Home Assistant", "pihole": "Pi-hole", "pi-hole": "Pi-hole",
|
|
"adguard": "AdGuard Home", "adguardhome": "AdGuard Home", "adguard home": "AdGuard Home",
|
|
"nginx proxy manager": "Nginx Proxy Manager", "npm": "Nginx Proxy Manager",
|
|
"paperless": "Paperless-ngx", "paperless-ngx": "Paperless-ngx", "wireguard": "WireGuard",
|
|
"tailscale": "Tailscale", "zerotier": "ZeroTier", "openvpn": "OpenVPN", "plex": "Plex",
|
|
"plex media server": "Plex", "jellyfin": "Jellyfin", "nextcloud": "Nextcloud", "immich": "Immich",
|
|
"vaultwarden": "Vaultwarden", "bitwarden": "Bitwarden", "samba": "Samba", "smb": "Samba",
|
|
"transmission": "Transmission", "cloudflare tunnel": "Cloudflare Tunnel",
|
|
"cloudflared": "Cloudflare Tunnel", "minecraft": "Minecraft server",
|
|
"minecraft server": "Minecraft server", "frigate": "Frigate", "portainer": "Portainer",
|
|
"syncthing": "Syncthing", "photoprism": "PhotoPrism", "uptime kuma": "Uptime Kuma",
|
|
"uptime-kuma": "Uptime Kuma", "opnsense": "OPNsense", "pfsense": "pfSense",
|
|
"sonarr": "Sonarr", "radarr": "Radarr", "prowlarr": "Prowlarr", "jackett": "Jackett",
|
|
"kodi": "Kodi", "emby": "Emby", "minidlna": "MiniDLNA", "dlna": "MiniDLNA",
|
|
}
|
|
|
|
|
|
# #11091 is Felhom's own announcement in the topic (found by the block-9 tagger): not a forum
|
|
# voice, so it is left out of every count and table.
|
|
EXCLUDE = {11091}
|
|
|
|
|
|
def split(v):
|
|
return [x.strip() for x in (v or "").split(";") if x.strip()]
|
|
|
|
|
|
def app_name(a):
|
|
return APP_ALIASES.get(a.strip().lower(), a.strip())
|
|
|
|
|
|
def hw_class(h):
|
|
return h.split("(")[0].strip().lower()
|
|
|
|
|
|
def load():
|
|
posts = {}
|
|
for line in open(os.path.join(CACHE, "posts.jsonl"), encoding="utf-8"):
|
|
p = json.loads(line)
|
|
posts[p["num"]] = p
|
|
tags = {}
|
|
for f in sorted(glob.glob(os.path.join(CACHE, "tags", "tags_*.csv"))):
|
|
for r in csv.DictReader(open(f, encoding="utf-8")):
|
|
n = int(r["num"])
|
|
if n in tags:
|
|
sys.exit("duplicate tag row #%d" % n)
|
|
tags[n] = r
|
|
# v2 re-tag (TAGGING-BRIEF.md "v2"): role, hardware, hardware_status replace the v1 values.
|
|
v2 = {}
|
|
for f in sorted(glob.glob(os.path.join(CACHE, "tags_v2", "v2_*.csv"))):
|
|
for r in csv.DictReader(open(f, encoding="utf-8")):
|
|
v2[int(r["num"])] = r
|
|
if set(v2) != set(tags):
|
|
sys.exit("v2 coverage differs from v1: %s" % sorted(set(v2) ^ set(tags))[:20])
|
|
for n, r in v2.items():
|
|
for c in ("role", "hardware", "hardware_status"):
|
|
tags[n][c] = r[c]
|
|
for n in EXCLUDE:
|
|
posts.pop(n, None)
|
|
tags.pop(n, None)
|
|
missing = sorted(set(posts) - set(tags))
|
|
extra = sorted(set(tags) - set(posts))
|
|
if missing or extra:
|
|
sys.exit("tag coverage broken: missing=%s extra=%s" % (missing[:20], extra[:20]))
|
|
return posts, tags
|
|
|
|
|
|
def threads(posts, tags):
|
|
asks = lambda n: n in tags and tags[n]["role"].endswith("-asks")
|
|
tid = {}
|
|
for n in posts:
|
|
cur, opener = n, None
|
|
seen = set()
|
|
while cur in posts and cur not in seen:
|
|
seen.add(cur)
|
|
if asks(cur):
|
|
if opener is None:
|
|
opener = cur
|
|
elif posts[cur]["author_id"] == posts[opener]["author_id"]:
|
|
opener = cur
|
|
else:
|
|
break
|
|
cur = posts[cur]["reply_to"]
|
|
if opener is not None:
|
|
tid[n] = opener
|
|
# An asker's follow-up that lost its reply link (no question_type, so it adds to an earlier
|
|
# question) joins that person's previous thread when it is at most 30 days older.
|
|
import datetime
|
|
day = lambda n: datetime.date.fromisoformat(posts[n]["date"][:10])
|
|
openers = sorted(set(tid.values()))
|
|
remap = {}
|
|
for o in openers:
|
|
if tags[o]["question_type"].strip():
|
|
continue
|
|
prev = [p for p in openers if p < o and posts[p]["author_id"] == posts[o]["author_id"]
|
|
and (day(o) - day(p)).days <= 30]
|
|
if prev:
|
|
remap[o] = remap.get(prev[-1], prev[-1])
|
|
return {n: remap.get(t, t) for n, t in tid.items()}
|
|
|
|
|
|
def people_by(rows, key):
|
|
c = collections.defaultdict(set)
|
|
for author, val in rows:
|
|
c[val].add(author)
|
|
return sorted(((len(v), k) for k, v in c.items()), key=lambda t: (-t[0], t[1]))
|
|
|
|
|
|
def table(title, ranked, top=30, examples=None):
|
|
print("\n### %s" % title)
|
|
print("| # | Value | People |" + (" Examples |" if examples else ""))
|
|
print("|---|---|---|" + ("---|" if examples else ""))
|
|
for i, (cnt, val) in enumerate(ranked[:top], 1):
|
|
ex = (" %s |" % ", ".join("#%d" % e for e in examples.get(val, [])[:3])) if examples else ""
|
|
print("| %d | %s | %d |%s" % (i, val, cnt, ex))
|
|
|
|
|
|
def main():
|
|
posts, tags = load()
|
|
if "--write-tags" in sys.argv:
|
|
with open(os.path.join(HERE, "tags.csv"), "w", encoding="utf-8", newline="") as f:
|
|
w = csv.writer(f)
|
|
w.writerow(COLS + ["thread"])
|
|
tid = threads(posts, tags)
|
|
for n in sorted(tags):
|
|
w.writerow([tags[n].get(c, "") for c in COLS] + [tid.get(n, "")])
|
|
print("wrote tags.csv (%d rows)" % len(tags))
|
|
tid = threads(posts, tags)
|
|
au = {n: posts[n]["author_id"] for n in posts}
|
|
role = {n: tags[n]["role"] for n in tags}
|
|
|
|
# --- 1. Numbers
|
|
openers = sorted(set(tid.values()))
|
|
beg_threads = [t for t in openers if role[t] == "beginner-asks"]
|
|
members = collections.defaultdict(list)
|
|
for n, t in tid.items():
|
|
members[t].append(n)
|
|
|
|
def resolved(t):
|
|
vals = [tags[n]["resolved"] for n in members[t] if au[n] == au[t] and tags[n]["resolved"]]
|
|
return "yes" if "yes" in vals else ("no" if "no" in vals else "unknown")
|
|
res = collections.Counter(resolved(t) for t in beg_threads)
|
|
# A beginner = a person with at least one post whose v2 role is beginner-asks (the column the
|
|
# spot-check agreed on at 97%); v1 beginner_evidence is kept in tags.csv as the reason.
|
|
beginners = {au[n] for n in tags if role[n] == "beginner-asks"}
|
|
print("## 1 Numbers")
|
|
print("posts=%d people=%d first=#%d last=#%d" % (len(posts), len(set(au.values())), min(posts), max(posts)))
|
|
print("roles: %s" % dict(collections.Counter(role.values())))
|
|
print("threads=%d beginner threads=%d beginner people=%d resolved=%s" %
|
|
(len(openers), len(beg_threads), len(beginners), dict(res)))
|
|
by_year = collections.Counter(tags[t]["year"] for t in beg_threads)
|
|
print("beginner threads by year: %s; posts by year: %s" %
|
|
(dict(sorted(by_year.items())), dict(sorted(collections.Counter(t["year"] for t in tags.values()).items()))))
|
|
|
|
# --- 2. Beginner question types (clusters come from clusters.json if present)
|
|
qt = collections.Counter()
|
|
for t in beg_threads:
|
|
for q in split(tags[t]["question_type"]):
|
|
qt[q] += 1
|
|
print("\n## 2 Beginner thread question types: %s" % qt.most_common())
|
|
cl_path = os.path.join(CACHE, "clusters.json")
|
|
if os.path.exists(cl_path):
|
|
cl = json.load(open(cl_path, encoding="utf-8"))
|
|
ranked = sorted(cl.items(), key=lambda kv: -len(kv[1]))
|
|
print("| # | Cluster | Threads | Examples |")
|
|
for i, (k, v) in enumerate(ranked, 1):
|
|
ts = [t for t in v if t in beg_threads]
|
|
print("| %d | %s | %d | %s |" % (i, k, len(ts), ", ".join("#%d" % t for t in ts[:3])))
|
|
rec = collections.Counter()
|
|
for t in ts:
|
|
for n in members[t]:
|
|
if role[n] != "answers":
|
|
continue
|
|
for col, st in (("platform", "platform_status"), ("hardware", "hardware_status"),
|
|
("apps", "apps_status")):
|
|
vals, sts = split(tags[n][col]), split(tags[n][st])
|
|
for j, x in enumerate(vals):
|
|
stx = sts[j] if len(sts) == len(vals) else (sts[0] if sts else "")
|
|
if stx == "recommends":
|
|
rec["%s:%s" % (col, hw_class(x) if col == "hardware" else app_name(x))] += 1
|
|
print(" answers recommended (posts): %s" % rec.most_common(8))
|
|
unassigned = [t for t in beg_threads if not any(t in v for v in cl.values())]
|
|
print("unassigned beginner threads: %d %s" % (len(unassigned), unassigned[:30]))
|
|
|
|
# --- 3. Apps
|
|
def ent_rows(col, status_col, status_pred, author_pred, norm=lambda x: x):
|
|
rows, ex = [], collections.defaultdict(list)
|
|
for n, t in tags.items():
|
|
if not author_pred(n):
|
|
continue
|
|
vals, sts = split(t[col]), split(t[status_col])
|
|
for i, v in enumerate(vals):
|
|
# Status cells are either one value for the whole post or one per item, aligned.
|
|
st = sts[i] if len(sts) == len(vals) else (sts[0] if sts else "")
|
|
if status_col != "role" and not status_pred(st):
|
|
continue
|
|
v = norm(v)
|
|
rows.append((au[n], v))
|
|
if n not in ex[v]:
|
|
ex[v].append(n)
|
|
return people_by(rows, 1), ex
|
|
is_beg = lambda n: au[n] in beginners
|
|
not_beg = lambda n: au[n] not in beginners
|
|
r, ex = ent_rows("apps", "apps_status", lambda s: s == "wants", is_beg, app_name)
|
|
table("3a Apps WANTED by beginners (people)", r, examples=ex)
|
|
r, ex = ent_rows("apps", "apps_status", lambda s: s == "runs", not_beg, app_name)
|
|
table("3b Apps RUN by experienced people (people)", r, examples=ex)
|
|
r, ex = ent_rows("apps", "apps_status", lambda s: s == "wants", lambda n: True, app_name)
|
|
table("3c Apps WANTED by anyone (people) - for the catalog gap", r, top=60, examples=ex)
|
|
r, ex = ent_rows("apps", "apps_status", lambda s: s == "recommends", lambda n: True, app_name)
|
|
table("3d Apps RECOMMENDED by anyone (people)", r, top=30, examples=ex)
|
|
|
|
# --- 4. Hardware
|
|
r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "owns", is_beg, hw_class)
|
|
table("4a Hardware OWNED by beginners (people)", r, examples=ex)
|
|
r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "plans", is_beg, hw_class)
|
|
table("4b Hardware PLANNED by beginners (people)", r, examples=ex)
|
|
r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "owns", lambda n: True, hw_class)
|
|
table("4c Hardware OWNED by anyone (people)", r, examples=ex)
|
|
r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "recommends", lambda n: True, hw_class)
|
|
table("4d Hardware RECOMMENDED (people)", r, examples=ex)
|
|
budgets = collections.defaultdict(list)
|
|
for n, t in tags.items():
|
|
try:
|
|
b = int(float(t["budget_huf"]))
|
|
except (ValueError, TypeError):
|
|
continue
|
|
if b > 0:
|
|
budgets[au[n]].append((n, b))
|
|
per_person = [max(v, key=lambda x: x[0])[1] for v in budgets.values()] # latest stated per person
|
|
beg_budget = [max(v, key=lambda x: x[0])[1] for a, v in budgets.items() if a in beginners]
|
|
for name, xs in (("all people", per_person), ("beginners", beg_budget)):
|
|
if xs:
|
|
print("budget %s: n=%d min=%d median=%d max=%d" % (name, len(xs), min(xs), statistics.median(xs), max(xs)))
|
|
|
|
# --- 5. Platforms
|
|
for title, pred, who in (("5a Platforms beginners want or tried", lambda s: s in ("wants", "tried", "runs"), is_beg),
|
|
("5b Platforms recommended by experienced people", lambda s: s == "recommends", not_beg),
|
|
("5c Platforms run by experienced people", lambda s: s == "runs", not_beg),
|
|
("5d Platforms advised against (anyone)", lambda s: s == "advises-against", lambda n: True)):
|
|
r, ex = ent_rows("platform", "platform_status", pred, who)
|
|
table(title, r, top=20, examples=ex)
|
|
tk = collections.defaultdict(lambda: collections.defaultdict(set))
|
|
tkex = collections.defaultdict(list)
|
|
for n, t in tags.items():
|
|
for x in split(t["turnkey"]):
|
|
name, _, tone = x.rpartition(":")
|
|
if not name:
|
|
name, tone = x, "0"
|
|
name = name.strip()
|
|
tk[name][tone.strip() or "0"].add(au[n])
|
|
tkex[name].append(n)
|
|
print("\n### 5e Turnkey products named (people by tone)")
|
|
print("| Product | People | + | 0 | - | Examples |")
|
|
print("|---|---|---|---|---|---|")
|
|
for name in sorted(tk, key=lambda k: -len(set().union(*tk[k].values()))):
|
|
allp = set().union(*tk[name].values())
|
|
print("| %s | %d | %d | %d | %d | %s |" % (name, len(allp), len(tk[name]["+"]), len(tk[name]["0"]),
|
|
len(tk[name]["-"]), ", ".join("#%d" % e for e in tkex[name][:3])))
|
|
|
|
# --- 6. Pains
|
|
r, ex = ent_rows("pain", "role", lambda s: True, lambda n: True)
|
|
table("6a Pains, everyone (people)", r, top=40, examples=ex)
|
|
r, ex = ent_rows("pain", "role", lambda s: True, is_beg)
|
|
table("6b Pains, beginners only (people)", r, top=20, examples=ex)
|
|
|
|
# --- 7. Trend: people per value per year for the top entities
|
|
print("\n## 7 Trend (people per year)")
|
|
years = ["2023", "2024", "2025", "2026"]
|
|
ppl_year = collections.Counter()
|
|
for n in tags:
|
|
pass
|
|
yr_people = {y: {au[n] for n in tags if tags[n]["year"] == y} for y in years}
|
|
print("active people per year: %s" % {y: len(v) for y, v in yr_people.items()})
|
|
for col, stat, norm in (("apps", None, app_name), ("hardware", None, hw_class), ("platform", None, lambda x: x),
|
|
("pain", None, lambda x: x)):
|
|
cnt = collections.defaultdict(lambda: collections.defaultdict(set))
|
|
for n, t in tags.items():
|
|
for v in split(t[col]):
|
|
cnt[norm(v)][t["year"]].add(au[n])
|
|
top = sorted(cnt, key=lambda k: -len(set().union(*cnt[k].values())))[:12]
|
|
print("\n%s: value | %s" % (col, " | ".join(years)))
|
|
for k in top:
|
|
print(" %s | %s" % (k, " | ".join("%d (%.0f%%)" % (len(cnt[k][y]), 100.0 * len(cnt[k][y]) / max(1, len(yr_people[y])))
|
|
for y in years)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|