admin
a7f897a7dd
gates / gates (push) Successful in 6m24s
Read-only. fetch.py + tables.py (stdlib), TABLE.md (45 charts, % of respondents), FELHOM.md (apps, features, reasons, hardware, trends, ten ranked recommendations), METHOD.md (hashes, counts, matching, MIT licence, limits). Survey files are not committed. One register row (R-989) and one STATUS line ask the operator to rule on the recommendations.
247 lines
11 KiB
Python
247 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Build TABLE.md from the cached selfh.st survey results (2023-2025).
|
||
|
||
Reads ~/research-cache/selfhst-survey/<year>-results.json (written by fetch.py).
|
||
Writes TABLE.md next to this script. Stdlib only.
|
||
|
||
Percent rule (spec §1, §8):
|
||
- multi-select charts: % of the year's respondent count;
|
||
- single-choice charts (marked "single"): % of the column sum, because the
|
||
question was asked to a subset or answered by fewer than all respondents.
|
||
"""
|
||
import json
|
||
import os
|
||
|
||
CACHE = os.path.expanduser("~/research-cache/selfhst-survey")
|
||
OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "TABLE.md")
|
||
YEARS = (2023, 2024, 2025)
|
||
PAGE = "https://selfh.st/survey/{}-results/"
|
||
DATA = "https://raw.githubusercontent.com/selfhst/cdn/main/assets/surveys/annual/{}-results.json"
|
||
|
||
# Respondent count per year. 2025: the results page states 4,081 completed responses.
|
||
# 2023/2024: the pages give only rounded numbers ("just shy of 1,900", "~3,700"); the exact
|
||
# figure is the column sum of the first, mandatory, single-choice question (Primary Purpose),
|
||
# which is 4,081 in 2025 too - the same method reproduces the stated 2025 number.
|
||
RESPONDENTS = {2023: 1869, 2024: 3655, 2025: 4081}
|
||
|
||
# 2025 title -> the 2023/2024 chart id it asks the same question as. None = 2025 only.
|
||
MATCH_2025 = {
|
||
"Primary Purpose": "personal-enterprise",
|
||
"Unique Users": "unique-users",
|
||
"Reasons for Self-Hosting": "why-self-host",
|
||
"Hardware Options": "hardware",
|
||
"Network-Attached Storage": "deploy-nas",
|
||
"Sensitive Data": "sensitive-data",
|
||
"Operating Systems": "operating-system",
|
||
"Backup Strategies": "backup-strategy",
|
||
"Cloud Storage Providers": "backup-cloud",
|
||
"Container Population": "deploy-containers",
|
||
"Containers vs Bare Metal": "prefer-containers",
|
||
# 2023 asked about Kubernetes only, 2024/2025 about any orchestration: 2023 kept separate.
|
||
"Container Orchestration": "orchestration-platform",
|
||
"Container Platforms": "container-platform",
|
||
"Management Software": "container-manager",
|
||
"Management Platforms": None,
|
||
"Update Methodology": "container-updates",
|
||
"Firewalls": "deploy-firewall",
|
||
"Remote Access": "remote-access",
|
||
"Reverse Proxies": "web-server",
|
||
"Registered Domains": "custom-domain",
|
||
"Domain Registrars": "domain-registrar",
|
||
"E-mail": "email-server",
|
||
"E-mail Components": "email-components",
|
||
"Database Deployment": "deploy-database",
|
||
"Media Streaming": "media-server",
|
||
"File Storage and Syncing": "file-sync",
|
||
"Authentication": None,
|
||
"Note-Taking / Documentation": "documentation-notes",
|
||
"Git": None,
|
||
"Home Automation": "smart-home",
|
||
"Game Servers": "deploy-game",
|
||
"Games": "game-server",
|
||
"Favorite Software": "favorite-app",
|
||
"Age Range": "age-range",
|
||
"Gender": "gender",
|
||
"Career Field": "employment-field",
|
||
"Mobile OS": "mobile-os",
|
||
"Donations": "donate-project",
|
||
"Education": "education-level",
|
||
}
|
||
# 2023's Kubernetes question differs from the later orchestration question.
|
||
NOT_MATCHED_2023 = {"orchestration-platform"}
|
||
|
||
# Single-choice charts -> % of column sum. Keyed by 2023/2024 id (or 2025 title when 2025-only).
|
||
SINGLE = {
|
||
"personal-enterprise", "unique-users", "deploy-nas", "sensitive-data", "offsite-backups",
|
||
"deploy-containers", "prefer-containers", "orchestration-platform", "container-manager",
|
||
"container-updates", "deploy-firewall", "custom-domain", "email-server", "stream-media",
|
||
"smart-home", "deploy-game", "host-activitypub", "age-range", "gender", "employment-field",
|
||
"mobile-os", "donate-project", "education-level",
|
||
}
|
||
|
||
# Old label -> 2025 label, per chart, where the wording changed but the answer is the same.
|
||
# Labels that differ only in case are merged automatically (and listed in the output).
|
||
ALIAS = {
|
||
"deploy-nas": {"Yes": "Deploy", "No": "Don't Deploy"},
|
||
"sensitive-data": {"Yes": "Avoid", "No": "Don't Avoid"},
|
||
"deploy-containers": {"Yes": "Deploy", "No": "Don't Deploy"},
|
||
"prefer-containers": {"Yes": "Containers", "No": "Bare Metal"},
|
||
"container-manager": {"Yes": "Use", "No": "Don't Use"},
|
||
"container-updates": {"Both": "Depends / Both"},
|
||
"deploy-firewall": {"Yes": "Non-ISP", "No": "ISP"},
|
||
"custom-domain": {"Yes": "Custom Domain", "No": "No Domain"},
|
||
"email-server": {"Yes": "Self-Host", "No": "Don't Self-Host"},
|
||
"smart-home": {"Yes": "Deploy", "No": "Don't Deploy"},
|
||
"deploy-game": {"Yes": "Host", "No": "Don't Host"},
|
||
"employment-field": {"Yes": "Technology", "No": "Non-Technology"},
|
||
"donate-project": {"Yes": "Donated", "No": "Haven't Donated"},
|
||
"gender": {"Man": "Male", "Woman": "Female"},
|
||
"education-level": {"Some college, no degree": "Some college"},
|
||
"age-range": {"10-19 years-old": "12-19 years-old"},
|
||
"operating-system": {"Generic Linux": "Linux"},
|
||
"hardware": {"Custom-built": "Custom build", "Enterprise": "Enterprise hardware",
|
||
"Mobile phone": "Mobile device"},
|
||
"deploy-database": {"Docker - Individual": "Container (Separate)",
|
||
"Docker - Shared": "Container (Shared)",
|
||
"Bare Metal - Individual": "Bare Metal (Separate)",
|
||
"Bare Metal - Shared": "Bare Metal (Shared)"},
|
||
"favorite-app": {"Paperless": "Paperless-ngx", "*arr Stack": "*arr", "Trilum": "Trilium"},
|
||
}
|
||
# Notes printed under a section heading.
|
||
NOTES = {
|
||
"unique-users": "2023 buckets 11-15 and 16-20 are kept apart; 2024/2025 ask 11-20.",
|
||
"prefer-containers": "2025 added a third answer (Depends); 2024 and 2025 asked only container users.",
|
||
"container-updates": "2025 was deployed as multi-select by mistake (column sum 4,113 > 3,930 asked).",
|
||
"age-range": "2023/2024 bucket 10-19 matched to 2025 12-19.",
|
||
"favorite-app": ("One answer per person, open text; each year omits small counts, so % is of all "
|
||
"respondents. 2023 'Paperless' matched to 'Paperless-ngx'; 2024 'Trilum' read as Trilium."),
|
||
"deploy-database": "Single-choice in 2023/2024, multi-select in 2025 (sum 5,004): % of respondents.",
|
||
"hardware": "2023 'Custom-built', 'Enterprise', 'Mobile phone' matched to the 2024/2025 wording.",
|
||
}
|
||
|
||
|
||
def charts_of(year):
|
||
path = os.path.join(CACHE, f"{year}-results.json")
|
||
if not os.path.exists(path):
|
||
return None
|
||
with open(path, encoding="utf-8") as f:
|
||
d = json.load(f)
|
||
if "charts" in d:
|
||
return d["charts"]
|
||
return [c for cat in d["categories"] for c in cat["charts"]]
|
||
|
||
|
||
def series(chart, alias):
|
||
out, merged = {}, []
|
||
canon = {}
|
||
for lab, val in zip(chart["labels"], chart["datasets"][0]["data"]):
|
||
lab = alias.get(lab, lab)
|
||
key = lab.lower()
|
||
if key in canon and canon[key] != lab:
|
||
merged.append(f"{lab} = {canon[key]}")
|
||
canon.setdefault(key, lab)
|
||
out[key] = out.get(key, 0) + val
|
||
return out, canon, merged
|
||
|
||
|
||
def pct(v, denom):
|
||
return f"{100.0 * v / denom:.1f}" if denom else "–"
|
||
|
||
|
||
def main():
|
||
per_year = {y: charts_of(y) for y in YEARS}
|
||
missing = [y for y in YEARS if per_year[y] is None]
|
||
by_id = {y: {c["id"]: c for c in (per_year[y] or [])} for y in (2023, 2024)}
|
||
|
||
# Build the section list: 2025 order first, then the 2023/2024 charts nobody matched.
|
||
sections = []
|
||
used = {2023: set(), 2024: set()}
|
||
for c25 in per_year[2025] or []:
|
||
cid = MATCH_2025.get(c25["title"])
|
||
c23 = by_id[2023].get(cid) if cid and cid not in NOT_MATCHED_2023 else None
|
||
c24 = by_id[2024].get(cid) if cid else None
|
||
for y, c in ((2023, c23), (2024, c24)):
|
||
if c:
|
||
used[y].add(c["id"])
|
||
sections.append({"key": cid or c25["title"], "charts": {2023: c23, 2024: c24, 2025: c25}})
|
||
unmatched = []
|
||
for cid in list(by_id[2023]) + [i for i in by_id[2024] if i not in by_id[2023]]:
|
||
c23 = by_id[2023].get(cid) if cid not in used[2023] else None
|
||
c24 = by_id[2024].get(cid) if cid not in used[2024] else None
|
||
if c23 or c24:
|
||
unmatched.append({"key": cid, "charts": {2023: c23, 2024: c24, 2025: None}})
|
||
|
||
L = ["# selfh.st self-hosting survey 2023–2025 — tables", "",
|
||
"Generated by `tables.py` from the files `fetch.py` downloads. Do not edit by hand.", "",
|
||
"| Year | Respondents | Results page | Data file |", "|---|---|---|---|"]
|
||
for y in YEARS:
|
||
L.append(f"| {y} | {RESPONDENTS[y]:,} | {PAGE.format(y)} | {DATA.format(y)} |")
|
||
L += ["", "Values are **% of the year's respondents** (multi-select: a column can pass 100 %).",
|
||
"Charts marked *single-choice* show **% of the column sum** instead.",
|
||
"`2025 n` is the raw 2025 response count. `–` = not asked that year (or not matched).", ""]
|
||
if missing:
|
||
L.append(f"**Missing years (download failed): {missing}.**\n")
|
||
|
||
def emit(num, sec, section_list_name):
|
||
ch = sec["charts"]
|
||
key = sec["key"]
|
||
single = key in SINGLE
|
||
alias = ALIAS.get(key, {})
|
||
titles = [f"{y}: {ch[y]['title']}" for y in YEARS if ch[y]]
|
||
rows, canon_all, merged_all, denom = {}, {}, [], {}
|
||
for y in reversed(YEARS): # 2025 first, so its spelling is the shown label
|
||
if not ch[y]:
|
||
continue
|
||
s, canon, merged = series(ch[y], alias)
|
||
merged_all += merged
|
||
for k, v in canon.items():
|
||
if k in canon_all and canon_all[k] != v:
|
||
merged_all.append(f"{y} '{v}' = '{canon_all[k]}'")
|
||
canon_all.setdefault(k, v)
|
||
denom[y] = sum(s.values()) if single else RESPONDENTS[y]
|
||
for k, v in s.items():
|
||
rows.setdefault(k, {})[y] = v
|
||
L.append(f"## T{num}. {ch[2025]['title'] if ch[2025] else ch[2024]['title'] if ch[2024] else ch[2023]['title']}")
|
||
L.append("")
|
||
L.append("Titles — " + " · ".join(titles))
|
||
L.append("")
|
||
if single:
|
||
L.append("*Single-choice: % of the column sum.*")
|
||
L.append("")
|
||
if key in NOTES:
|
||
L.append(f"Note: {NOTES[key]}")
|
||
L.append("")
|
||
if merged_all:
|
||
L.append("Merged spelling/case variants: " + "; ".join(sorted(set(merged_all))))
|
||
L.append("")
|
||
L.append("| label | 2023 | 2024 | 2025 | 2025 n |")
|
||
L.append("|---|---|---|---|---|")
|
||
|
||
def sortkey(k):
|
||
r = rows[k]
|
||
return (k == "other", -r.get(2025, -1), -r.get(2024, -1), -r.get(2023, -1))
|
||
for k in sorted(rows, key=sortkey):
|
||
r = rows[k]
|
||
cells = [pct(r[y], denom[y]) if y in r else "–" for y in YEARS]
|
||
n = f"{r[2025]:,}" if 2025 in r else "–"
|
||
L.append(f"| {canon_all[k]} | {' | '.join(cells)} | {n} |")
|
||
L.append("")
|
||
|
||
n = 0
|
||
for sec in sections:
|
||
n += 1
|
||
emit(n, sec, "matched")
|
||
L += ["# Unmatched charts (asked in 2023 and/or 2024, no same question in 2025)", ""]
|
||
for sec in unmatched:
|
||
n += 1
|
||
emit(n, sec, "unmatched")
|
||
|
||
with open(OUT, "w", encoding="utf-8") as f:
|
||
f.write("\n".join(L).rstrip() + "\n")
|
||
print(f"wrote {OUT}: {n} sections ({len(sections)} from 2025, {len(unmatched)} unmatched)")
|
||
return 1 if missing else 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|