#!/usr/bin/env python3
"""Merge the helper tags, build threads, write tags.csv (committed) and print the FINDINGS tables.

Read-only research (forum-research-2026-10-12). Stdlib only.
Inputs (cache, never in git): posts.jsonl, tags/tags_*.csv, clusters.json (optional).
Output: tags.csv here (num + tags only: no text, no author_id, no gist) and tables on stdout.

Threads (spec 7 C): walk up reply_to from a post; the first ancestor-or-self whose role ends in
"-asks" opens a thread; keep walking while the next "-asks" ancestor is by the SAME person (a
follow-up question is the same thread). A post with no "-asks" ancestor belongs to no thread.
A thread is a beginner thread when its opening post is beginner-asks.
Thread resolved = yes if any post by the asker in it is tagged yes, else no if any is no, else unknown.

"People" = distinct author_id. Usage: python3 build.py [--write-tags]
"""
import collections
import csv
import glob
import json
import os
import statistics
import sys

HERE = os.path.dirname(os.path.abspath(__file__))
CACHE = os.path.expanduser("~/research-cache/prohardver-hazi-szerver")
COLS = ["num", "year", "role", "beginner_evidence", "question_type", "hardware", "hardware_status",
        "apps", "apps_status", "platform", "platform_status", "pain", "budget_huf", "resolved", "turnkey"]

# Spelling variants the taggers produced -> one canonical name (filled from the merged tag values).
APP_ALIASES = {
    "qbittorrent": "qBittorrent", "qbit": "qBittorrent", "homeassistant": "Home Assistant",
    "home assistant os": "Home Assistant", "pihole": "Pi-hole", "pi-hole": "Pi-hole",
    "adguard": "AdGuard Home", "adguardhome": "AdGuard Home", "adguard home": "AdGuard Home",
    "nginx proxy manager": "Nginx Proxy Manager", "npm": "Nginx Proxy Manager",
    "paperless": "Paperless-ngx", "paperless-ngx": "Paperless-ngx", "wireguard": "WireGuard",
    "tailscale": "Tailscale", "zerotier": "ZeroTier", "openvpn": "OpenVPN", "plex": "Plex",
    "plex media server": "Plex", "jellyfin": "Jellyfin", "nextcloud": "Nextcloud", "immich": "Immich",
    "vaultwarden": "Vaultwarden", "bitwarden": "Bitwarden", "samba": "Samba", "smb": "Samba",
    "transmission": "Transmission", "cloudflare tunnel": "Cloudflare Tunnel",
    "cloudflared": "Cloudflare Tunnel", "minecraft": "Minecraft server",
    "minecraft server": "Minecraft server", "frigate": "Frigate", "portainer": "Portainer",
    "syncthing": "Syncthing", "photoprism": "PhotoPrism", "uptime kuma": "Uptime Kuma",
    "uptime-kuma": "Uptime Kuma", "opnsense": "OPNsense", "pfsense": "pfSense",
    "sonarr": "Sonarr", "radarr": "Radarr", "prowlarr": "Prowlarr", "jackett": "Jackett",
    "kodi": "Kodi", "emby": "Emby", "minidlna": "MiniDLNA", "dlna": "MiniDLNA",
}


# #11091 is Felhom's own announcement in the topic (found by the block-9 tagger): not a forum
# voice, so it is left out of every count and table.
EXCLUDE = {11091}


def split(v):
    return [x.strip() for x in (v or "").split(";") if x.strip()]


def app_name(a):
    return APP_ALIASES.get(a.strip().lower(), a.strip())


def hw_class(h):
    return h.split("(")[0].strip().lower()


def load():
    posts = {}
    for line in open(os.path.join(CACHE, "posts.jsonl"), encoding="utf-8"):
        p = json.loads(line)
        posts[p["num"]] = p
    tags = {}
    for f in sorted(glob.glob(os.path.join(CACHE, "tags", "tags_*.csv"))):
        for r in csv.DictReader(open(f, encoding="utf-8")):
            n = int(r["num"])
            if n in tags:
                sys.exit("duplicate tag row #%d" % n)
            tags[n] = r
    # v2 re-tag (TAGGING-BRIEF.md "v2"): role, hardware, hardware_status replace the v1 values.
    v2 = {}
    for f in sorted(glob.glob(os.path.join(CACHE, "tags_v2", "v2_*.csv"))):
        for r in csv.DictReader(open(f, encoding="utf-8")):
            v2[int(r["num"])] = r
    if set(v2) != set(tags):
        sys.exit("v2 coverage differs from v1: %s" % sorted(set(v2) ^ set(tags))[:20])
    for n, r in v2.items():
        for c in ("role", "hardware", "hardware_status"):
            tags[n][c] = r[c]
    for n in EXCLUDE:
        posts.pop(n, None)
        tags.pop(n, None)
    missing = sorted(set(posts) - set(tags))
    extra = sorted(set(tags) - set(posts))
    if missing or extra:
        sys.exit("tag coverage broken: missing=%s extra=%s" % (missing[:20], extra[:20]))
    return posts, tags


def threads(posts, tags):
    asks = lambda n: n in tags and tags[n]["role"].endswith("-asks")
    tid = {}
    for n in posts:
        cur, opener = n, None
        seen = set()
        while cur in posts and cur not in seen:
            seen.add(cur)
            if asks(cur):
                if opener is None:
                    opener = cur
                elif posts[cur]["author_id"] == posts[opener]["author_id"]:
                    opener = cur
                else:
                    break
            cur = posts[cur]["reply_to"]
        if opener is not None:
            tid[n] = opener
    # An asker's follow-up that lost its reply link (no question_type, so it adds to an earlier
    # question) joins that person's previous thread when it is at most 30 days older.
    import datetime
    day = lambda n: datetime.date.fromisoformat(posts[n]["date"][:10])
    openers = sorted(set(tid.values()))
    remap = {}
    for o in openers:
        if tags[o]["question_type"].strip():
            continue
        prev = [p for p in openers if p < o and posts[p]["author_id"] == posts[o]["author_id"]
                and (day(o) - day(p)).days <= 30]
        if prev:
            remap[o] = remap.get(prev[-1], prev[-1])
    return {n: remap.get(t, t) for n, t in tid.items()}


def people_by(rows, key):
    c = collections.defaultdict(set)
    for author, val in rows:
        c[val].add(author)
    return sorted(((len(v), k) for k, v in c.items()), key=lambda t: (-t[0], t[1]))


def table(title, ranked, top=30, examples=None):
    print("\n### %s" % title)
    print("| # | Value | People |" + (" Examples |" if examples else ""))
    print("|---|---|---|" + ("---|" if examples else ""))
    for i, (cnt, val) in enumerate(ranked[:top], 1):
        ex = (" %s |" % ", ".join("#%d" % e for e in examples.get(val, [])[:3])) if examples else ""
        print("| %d | %s | %d |%s" % (i, val, cnt, ex))


def main():
    posts, tags = load()
    if "--write-tags" in sys.argv:
        with open(os.path.join(HERE, "tags.csv"), "w", encoding="utf-8", newline="") as f:
            w = csv.writer(f)
            w.writerow(COLS + ["thread"])
            tid = threads(posts, tags)
            for n in sorted(tags):
                w.writerow([tags[n].get(c, "") for c in COLS] + [tid.get(n, "")])
        print("wrote tags.csv (%d rows)" % len(tags))
    tid = threads(posts, tags)
    au = {n: posts[n]["author_id"] for n in posts}
    role = {n: tags[n]["role"] for n in tags}

    # --- 1. Numbers
    openers = sorted(set(tid.values()))
    beg_threads = [t for t in openers if role[t] == "beginner-asks"]
    members = collections.defaultdict(list)
    for n, t in tid.items():
        members[t].append(n)

    def resolved(t):
        vals = [tags[n]["resolved"] for n in members[t] if au[n] == au[t] and tags[n]["resolved"]]
        return "yes" if "yes" in vals else ("no" if "no" in vals else "unknown")
    res = collections.Counter(resolved(t) for t in beg_threads)
    # A beginner = a person with at least one post whose v2 role is beginner-asks (the column the
    # spot-check agreed on at 97%); v1 beginner_evidence is kept in tags.csv as the reason.
    beginners = {au[n] for n in tags if role[n] == "beginner-asks"}
    print("## 1 Numbers")
    print("posts=%d people=%d first=#%d last=#%d" % (len(posts), len(set(au.values())), min(posts), max(posts)))
    print("roles: %s" % dict(collections.Counter(role.values())))
    print("threads=%d beginner threads=%d beginner people=%d resolved=%s" %
          (len(openers), len(beg_threads), len(beginners), dict(res)))
    by_year = collections.Counter(tags[t]["year"] for t in beg_threads)
    print("beginner threads by year: %s; posts by year: %s" %
          (dict(sorted(by_year.items())), dict(sorted(collections.Counter(t["year"] for t in tags.values()).items()))))

    # --- 2. Beginner question types (clusters come from clusters.json if present)
    qt = collections.Counter()
    for t in beg_threads:
        for q in split(tags[t]["question_type"]):
            qt[q] += 1
    print("\n## 2 Beginner thread question types: %s" % qt.most_common())
    cl_path = os.path.join(CACHE, "clusters.json")
    if os.path.exists(cl_path):
        cl = json.load(open(cl_path, encoding="utf-8"))
        ranked = sorted(cl.items(), key=lambda kv: -len(kv[1]))
        print("| # | Cluster | Threads | Examples |")
        for i, (k, v) in enumerate(ranked, 1):
            ts = [t for t in v if t in beg_threads]
            print("| %d | %s | %d | %s |" % (i, k, len(ts), ", ".join("#%d" % t for t in ts[:3])))
            rec = collections.Counter()
            for t in ts:
                for n in members[t]:
                    if role[n] != "answers":
                        continue
                    for col, st in (("platform", "platform_status"), ("hardware", "hardware_status"),
                                    ("apps", "apps_status")):
                        vals, sts = split(tags[n][col]), split(tags[n][st])
                        for j, x in enumerate(vals):
                            stx = sts[j] if len(sts) == len(vals) else (sts[0] if sts else "")
                            if stx == "recommends":
                                rec["%s:%s" % (col, hw_class(x) if col == "hardware" else app_name(x))] += 1
            print("    answers recommended (posts): %s" % rec.most_common(8))
        unassigned = [t for t in beg_threads if not any(t in v for v in cl.values())]
        print("unassigned beginner threads: %d %s" % (len(unassigned), unassigned[:30]))

    # --- 3. Apps
    def ent_rows(col, status_col, status_pred, author_pred, norm=lambda x: x):
        rows, ex = [], collections.defaultdict(list)
        for n, t in tags.items():
            if not author_pred(n):
                continue
            vals, sts = split(t[col]), split(t[status_col])
            for i, v in enumerate(vals):
                # Status cells are either one value for the whole post or one per item, aligned.
                st = sts[i] if len(sts) == len(vals) else (sts[0] if sts else "")
                if status_col != "role" and not status_pred(st):
                    continue
                v = norm(v)
                rows.append((au[n], v))
                if n not in ex[v]:
                    ex[v].append(n)
        return people_by(rows, 1), ex
    is_beg = lambda n: au[n] in beginners
    not_beg = lambda n: au[n] not in beginners
    r, ex = ent_rows("apps", "apps_status", lambda s: s == "wants", is_beg, app_name)
    table("3a Apps WANTED by beginners (people)", r, examples=ex)
    r, ex = ent_rows("apps", "apps_status", lambda s: s == "runs", not_beg, app_name)
    table("3b Apps RUN by experienced people (people)", r, examples=ex)
    r, ex = ent_rows("apps", "apps_status", lambda s: s == "wants", lambda n: True, app_name)
    table("3c Apps WANTED by anyone (people) - for the catalog gap", r, top=60, examples=ex)
    r, ex = ent_rows("apps", "apps_status", lambda s: s == "recommends", lambda n: True, app_name)
    table("3d Apps RECOMMENDED by anyone (people)", r, top=30, examples=ex)

    # --- 4. Hardware
    r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "owns", is_beg, hw_class)
    table("4a Hardware OWNED by beginners (people)", r, examples=ex)
    r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "plans", is_beg, hw_class)
    table("4b Hardware PLANNED by beginners (people)", r, examples=ex)
    r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "owns", lambda n: True, hw_class)
    table("4c Hardware OWNED by anyone (people)", r, examples=ex)
    r, ex = ent_rows("hardware", "hardware_status", lambda s: s == "recommends", lambda n: True, hw_class)
    table("4d Hardware RECOMMENDED (people)", r, examples=ex)
    budgets = collections.defaultdict(list)
    for n, t in tags.items():
        try:
            b = int(float(t["budget_huf"]))
        except (ValueError, TypeError):
            continue
        if b > 0:
            budgets[au[n]].append((n, b))
    per_person = [max(v, key=lambda x: x[0])[1] for v in budgets.values()]  # latest stated per person
    beg_budget = [max(v, key=lambda x: x[0])[1] for a, v in budgets.items() if a in beginners]
    for name, xs in (("all people", per_person), ("beginners", beg_budget)):
        if xs:
            print("budget %s: n=%d min=%d median=%d max=%d" % (name, len(xs), min(xs), statistics.median(xs), max(xs)))

    # --- 5. Platforms
    for title, pred, who in (("5a Platforms beginners want or tried", lambda s: s in ("wants", "tried", "runs"), is_beg),
                             ("5b Platforms recommended by experienced people", lambda s: s == "recommends", not_beg),
                             ("5c Platforms run by experienced people", lambda s: s == "runs", not_beg),
                             ("5d Platforms advised against (anyone)", lambda s: s == "advises-against", lambda n: True)):
        r, ex = ent_rows("platform", "platform_status", pred, who)
        table(title, r, top=20, examples=ex)
    tk = collections.defaultdict(lambda: collections.defaultdict(set))
    tkex = collections.defaultdict(list)
    for n, t in tags.items():
        for x in split(t["turnkey"]):
            name, _, tone = x.rpartition(":")
            if not name:
                name, tone = x, "0"
            name = name.strip()
            tk[name][tone.strip() or "0"].add(au[n])
            tkex[name].append(n)
    print("\n### 5e Turnkey products named (people by tone)")
    print("| Product | People | + | 0 | - | Examples |")
    print("|---|---|---|---|---|---|")
    for name in sorted(tk, key=lambda k: -len(set().union(*tk[k].values()))):
        allp = set().union(*tk[name].values())
        print("| %s | %d | %d | %d | %d | %s |" % (name, len(allp), len(tk[name]["+"]), len(tk[name]["0"]),
                                                  len(tk[name]["-"]), ", ".join("#%d" % e for e in tkex[name][:3])))

    # --- 6. Pains
    r, ex = ent_rows("pain", "role", lambda s: True, lambda n: True)
    table("6a Pains, everyone (people)", r, top=40, examples=ex)
    r, ex = ent_rows("pain", "role", lambda s: True, is_beg)
    table("6b Pains, beginners only (people)", r, top=20, examples=ex)

    # --- 7. Trend: people per value per year for the top entities
    print("\n## 7 Trend (people per year)")
    years = ["2023", "2024", "2025", "2026"]
    ppl_year = collections.Counter()
    for n in tags:
        pass
    yr_people = {y: {au[n] for n in tags if tags[n]["year"] == y} for y in years}
    print("active people per year: %s" % {y: len(v) for y, v in yr_people.items()})
    for col, stat, norm in (("apps", None, app_name), ("hardware", None, hw_class), ("platform", None, lambda x: x),
                            ("pain", None, lambda x: x)):
        cnt = collections.defaultdict(lambda: collections.defaultdict(set))
        for n, t in tags.items():
            for v in split(t[col]):
                cnt[norm(v)][t["year"]].add(au[n])
        top = sorted(cnt, key=lambda k: -len(set().union(*cnt[k].values())))[:12]
        print("\n%s: value | %s" % (col, " | ".join(years)))
        for k in top:
            print("  %s | %s" % (k, " | ".join("%d (%.0f%%)" % (len(cnt[k][y]), 100.0 * len(cnt[k][y]) / max(1, len(yr_people[y])))
                                               for y in years)))


if __name__ == "__main__":
    main()