#!/usr/bin/env python3
"""Build TABLE.md from the cached selfh.st survey results (2023-2025).

Reads ~/research-cache/selfhst-survey/<year>-results.json (written by fetch.py).
Writes TABLE.md next to this script. Stdlib only.

Percent rule (spec §1, §8):
- multi-select charts: % of the year's respondent count;
- single-choice charts (marked "single"): % of the column sum, because the
  question was asked to a subset or answered by fewer than all respondents.
"""
import json
import os

CACHE = os.path.expanduser("~/research-cache/selfhst-survey")
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "TABLE.md")
YEARS = (2023, 2024, 2025)
PAGE = "https://selfh.st/survey/{}-results/"
DATA = "https://raw.githubusercontent.com/selfhst/cdn/main/assets/surveys/annual/{}-results.json"

# Respondent count per year. 2025: the results page states 4,081 completed responses.
# 2023/2024: the pages give only rounded numbers ("just shy of 1,900", "~3,700"); the exact
# figure is the column sum of the first, mandatory, single-choice question (Primary Purpose),
# which is 4,081 in 2025 too - the same method reproduces the stated 2025 number.
RESPONDENTS = {2023: 1869, 2024: 3655, 2025: 4081}

# 2025 title -> the 2023/2024 chart id it asks the same question as. None = 2025 only.
MATCH_2025 = {
    "Primary Purpose": "personal-enterprise",
    "Unique Users": "unique-users",
    "Reasons for Self-Hosting": "why-self-host",
    "Hardware Options": "hardware",
    "Network-Attached Storage": "deploy-nas",
    "Sensitive Data": "sensitive-data",
    "Operating Systems": "operating-system",
    "Backup Strategies": "backup-strategy",
    "Cloud Storage Providers": "backup-cloud",
    "Container Population": "deploy-containers",
    "Containers vs Bare Metal": "prefer-containers",
    # 2023 asked about Kubernetes only, 2024/2025 about any orchestration: 2023 kept separate.
    "Container Orchestration": "orchestration-platform",
    "Container Platforms": "container-platform",
    "Management Software": "container-manager",
    "Management Platforms": None,
    "Update Methodology": "container-updates",
    "Firewalls": "deploy-firewall",
    "Remote Access": "remote-access",
    "Reverse Proxies": "web-server",
    "Registered Domains": "custom-domain",
    "Domain Registrars": "domain-registrar",
    "E-mail": "email-server",
    "E-mail Components": "email-components",
    "Database Deployment": "deploy-database",
    "Media Streaming": "media-server",
    "File Storage and Syncing": "file-sync",
    "Authentication": None,
    "Note-Taking / Documentation": "documentation-notes",
    "Git": None,
    "Home Automation": "smart-home",
    "Game Servers": "deploy-game",
    "Games": "game-server",
    "Favorite Software": "favorite-app",
    "Age Range": "age-range",
    "Gender": "gender",
    "Career Field": "employment-field",
    "Mobile OS": "mobile-os",
    "Donations": "donate-project",
    "Education": "education-level",
}
# 2023's Kubernetes question differs from the later orchestration question.
NOT_MATCHED_2023 = {"orchestration-platform"}

# Single-choice charts -> % of column sum. Keyed by 2023/2024 id (or 2025 title when 2025-only).
SINGLE = {
    "personal-enterprise", "unique-users", "deploy-nas", "sensitive-data", "offsite-backups",
    "deploy-containers", "prefer-containers", "orchestration-platform", "container-manager",
    "container-updates", "deploy-firewall", "custom-domain", "email-server", "stream-media",
    "smart-home", "deploy-game", "host-activitypub", "age-range", "gender", "employment-field",
    "mobile-os", "donate-project", "education-level",
}

# Old label -> 2025 label, per chart, where the wording changed but the answer is the same.
# Labels that differ only in case are merged automatically (and listed in the output).
ALIAS = {
    "deploy-nas": {"Yes": "Deploy", "No": "Don't Deploy"},
    "sensitive-data": {"Yes": "Avoid", "No": "Don't Avoid"},
    "deploy-containers": {"Yes": "Deploy", "No": "Don't Deploy"},
    "prefer-containers": {"Yes": "Containers", "No": "Bare Metal"},
    "container-manager": {"Yes": "Use", "No": "Don't Use"},
    "container-updates": {"Both": "Depends / Both"},
    "deploy-firewall": {"Yes": "Non-ISP", "No": "ISP"},
    "custom-domain": {"Yes": "Custom Domain", "No": "No Domain"},
    "email-server": {"Yes": "Self-Host", "No": "Don't Self-Host"},
    "smart-home": {"Yes": "Deploy", "No": "Don't Deploy"},
    "deploy-game": {"Yes": "Host", "No": "Don't Host"},
    "employment-field": {"Yes": "Technology", "No": "Non-Technology"},
    "donate-project": {"Yes": "Donated", "No": "Haven't Donated"},
    "gender": {"Man": "Male", "Woman": "Female"},
    "education-level": {"Some college, no degree": "Some college"},
    "age-range": {"10-19 years-old": "12-19 years-old"},
    "operating-system": {"Generic Linux": "Linux"},
    "hardware": {"Custom-built": "Custom build", "Enterprise": "Enterprise hardware",
                 "Mobile phone": "Mobile device"},
    "deploy-database": {"Docker - Individual": "Container (Separate)",
                        "Docker - Shared": "Container (Shared)",
                        "Bare Metal - Individual": "Bare Metal (Separate)",
                        "Bare Metal - Shared": "Bare Metal (Shared)"},
    "favorite-app": {"Paperless": "Paperless-ngx", "*arr Stack": "*arr", "Trilum": "Trilium"},
}
# Notes printed under a section heading.
NOTES = {
    "unique-users": "2023 buckets 11-15 and 16-20 are kept apart; 2024/2025 ask 11-20.",
    "prefer-containers": "2025 added a third answer (Depends); 2024 and 2025 asked only container users.",
    "container-updates": "2025 was deployed as multi-select by mistake (column sum 4,113 > 3,930 asked).",
    "age-range": "2023/2024 bucket 10-19 matched to 2025 12-19.",
    "favorite-app": ("One answer per person, open text; each year omits small counts, so % is of all "
                     "respondents. 2023 'Paperless' matched to 'Paperless-ngx'; 2024 'Trilum' read as Trilium."),
    "deploy-database": "Single-choice in 2023/2024, multi-select in 2025 (sum 5,004): % of respondents.",
    "hardware": "2023 'Custom-built', 'Enterprise', 'Mobile phone' matched to the 2024/2025 wording.",
}


def charts_of(year):
    path = os.path.join(CACHE, f"{year}-results.json")
    if not os.path.exists(path):
        return None
    with open(path, encoding="utf-8") as f:
        d = json.load(f)
    if "charts" in d:
        return d["charts"]
    return [c for cat in d["categories"] for c in cat["charts"]]


def series(chart, alias):
    out, merged = {}, []
    canon = {}
    for lab, val in zip(chart["labels"], chart["datasets"][0]["data"]):
        lab = alias.get(lab, lab)
        key = lab.lower()
        if key in canon and canon[key] != lab:
            merged.append(f"{lab} = {canon[key]}")
        canon.setdefault(key, lab)
        out[key] = out.get(key, 0) + val
    return out, canon, merged


def pct(v, denom):
    return f"{100.0 * v / denom:.1f}" if denom else "–"


def main():
    per_year = {y: charts_of(y) for y in YEARS}
    missing = [y for y in YEARS if per_year[y] is None]
    by_id = {y: {c["id"]: c for c in (per_year[y] or [])} for y in (2023, 2024)}

    # Build the section list: 2025 order first, then the 2023/2024 charts nobody matched.
    sections = []
    used = {2023: set(), 2024: set()}
    for c25 in per_year[2025] or []:
        cid = MATCH_2025.get(c25["title"])
        c23 = by_id[2023].get(cid) if cid and cid not in NOT_MATCHED_2023 else None
        c24 = by_id[2024].get(cid) if cid else None
        for y, c in ((2023, c23), (2024, c24)):
            if c:
                used[y].add(c["id"])
        sections.append({"key": cid or c25["title"], "charts": {2023: c23, 2024: c24, 2025: c25}})
    unmatched = []
    for cid in list(by_id[2023]) + [i for i in by_id[2024] if i not in by_id[2023]]:
        c23 = by_id[2023].get(cid) if cid not in used[2023] else None
        c24 = by_id[2024].get(cid) if cid not in used[2024] else None
        if c23 or c24:
            unmatched.append({"key": cid, "charts": {2023: c23, 2024: c24, 2025: None}})

    L = ["# selfh.st self-hosting survey 2023–2025 — tables", "",
         "Generated by `tables.py` from the files `fetch.py` downloads. Do not edit by hand.", "",
         "| Year | Respondents | Results page | Data file |", "|---|---|---|---|"]
    for y in YEARS:
        L.append(f"| {y} | {RESPONDENTS[y]:,} | {PAGE.format(y)} | {DATA.format(y)} |")
    L += ["", "Values are **% of the year's respondents** (multi-select: a column can pass 100 %).",
          "Charts marked *single-choice* show **% of the column sum** instead.",
          "`2025 n` is the raw 2025 response count. `–` = not asked that year (or not matched).", ""]
    if missing:
        L.append(f"**Missing years (download failed): {missing}.**\n")

    def emit(num, sec, section_list_name):
        ch = sec["charts"]
        key = sec["key"]
        single = key in SINGLE
        alias = ALIAS.get(key, {})
        titles = [f"{y}: {ch[y]['title']}" for y in YEARS if ch[y]]
        rows, canon_all, merged_all, denom = {}, {}, [], {}
        for y in reversed(YEARS):  # 2025 first, so its spelling is the shown label
            if not ch[y]:
                continue
            s, canon, merged = series(ch[y], alias)
            merged_all += merged
            for k, v in canon.items():
                if k in canon_all and canon_all[k] != v:
                    merged_all.append(f"{y} '{v}' = '{canon_all[k]}'")
                canon_all.setdefault(k, v)
            denom[y] = sum(s.values()) if single else RESPONDENTS[y]
            for k, v in s.items():
                rows.setdefault(k, {})[y] = v
        L.append(f"## T{num}. {ch[2025]['title'] if ch[2025] else ch[2024]['title'] if ch[2024] else ch[2023]['title']}")
        L.append("")
        L.append("Titles — " + " · ".join(titles))
        L.append("")
        if single:
            L.append("*Single-choice: % of the column sum.*")
            L.append("")
        if key in NOTES:
            L.append(f"Note: {NOTES[key]}")
            L.append("")
        if merged_all:
            L.append("Merged spelling/case variants: " + "; ".join(sorted(set(merged_all))))
            L.append("")
        L.append("| label | 2023 | 2024 | 2025 | 2025 n |")
        L.append("|---|---|---|---|---|")

        def sortkey(k):
            r = rows[k]
            return (k == "other", -r.get(2025, -1), -r.get(2024, -1), -r.get(2023, -1))
        for k in sorted(rows, key=sortkey):
            r = rows[k]
            cells = [pct(r[y], denom[y]) if y in r else "–" for y in YEARS]
            n = f"{r[2025]:,}" if 2025 in r else "–"
            L.append(f"| {canon_all[k]} | {' | '.join(cells)} | {n} |")
        L.append("")

    n = 0
    for sec in sections:
        n += 1
        emit(n, sec, "matched")
    L += ["# Unmatched charts (asked in 2023 and/or 2024, no same question in 2025)", ""]
    for sec in unmatched:
        n += 1
        emit(n, sec, "unmatched")

    with open(OUT, "w", encoding="utf-8") as f:
        f.write("\n".join(L).rstrip() + "\n")
    print(f"wrote {OUT}: {n} sections ({len(sections)} from 2025, {len(unmatched)} unmatched)")
    return 1 if missing else 0


if __name__ == "__main__":
    raise SystemExit(main())