app-catalog-felhom.eu/scripts/check-onboarding.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""check-onboarding.py — a NEW catalog app carries a complete onboarding record (NEW-APP-CHECKLIST.md).

WHY. Operator request 2026-10-01: before a new app is offered, it is mapped and tested against one checklist —
storage, database, the first admin, health checks, memory, updates, mail, the household's text. A checklist that
nothing enforces is a wish; this gate is the enforcement. It reads files only (no network, no containers, no git
history), so it runs in `catalog_gates.py --fast`: the pre-push hook and CI.

THE RULE, for every directory under templates/ that is NOT in EXEMPT:
  1. `onboarding/<app>.md` exists, its `app:` line names the app, and its `opened: YYYY-MM-DD` is a real date
     on or after CUTOFF and not in the future.
  2. It answers every checklist id whose `since` date is on or before `opened:` (an id added later binds only
     apps opened after it — the checklist stores the date per id).
  3. No answer is `open`.
  4. Every `done` names evidence that EXISTS: a non-empty file, or a directory holding at least one non-empty
     file. An empty directory is a label, not evidence (R-410: a `mkdir` once turned a release gate green).
  5. Every `n/a` carries a reason of at least MIN_REASON_WORDS words.
For an EXEMPT app that has a record anyway (wger, the pilot): the record must be well-formed (known ids, no
duplicate, a valid status, `done` evidence that exists, `n/a` with a reason) — `open` and missing ids are allowed.
And `onboarding/_TEMPLATE.md` must carry every checklist id, so a row added to the checklist cannot be forgotten
in the template a new app copies.

EVIDENCE PATHS are written from the workspace root. `app-catalog-felhom.eu/…` resolves against THIS checkout
(whatever its directory is called — the CI runner checks out into another name). Any other first component
(`felhom.eu/…`) resolves against the checkout's parent directory; if that sibling repository is not there (the CI
runner fetches this repo alone) the path is NOT CHECKED and the count is printed — the pre-push hook on DooPlex has
the sibling and checks it. Same shape as engine-major's shallow-clone skip: said out loud, never silent.

WHY THE 53 ARE LISTED BY NAME and not "new since commit X": the CI runner fetches at --depth 1 and has no history
to diff (R-452), and a list is a fact a reader can check. A directory not on the list is new, whatever its age.

Run from the repo root:  python3 scripts/check-onboarding.py [--root=DIR] [--today=YYYY-MM-DD]
Exit 0 all records complete · 1 a record is missing or incomplete · 2 the checklist itself cannot be read.
"""
import datetime
import os
import re
import sys

# A gate's own console must not decide its verdict. The checklist titles this gate quotes back carry
# non-ASCII characters (an arrow in 3.2/3.3, accented Hungarian), and a Windows console here defaults
# to cp1250: on 2026-10-10 this gate hit UnicodeEncodeError while printing WHICH ids were still open,
# so an informative exit 1 arrived as a traceback. Same trap class as poster_facts_gate.py's.
for _stream in (sys.stdout, sys.stderr):
    try:
        _stream.reconfigure(encoding="utf-8", errors="replace")
    except (AttributeError, ValueError, OSError):  # pragma: no cover - old Python, or a pipe
        pass

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CATALOG_PREFIX = "app-catalog-felhom.eu/"
CUTOFF = "2026-10-01"
MIN_REASON_WORDS = 4
STATUSES = ("done", "n/a", "open")

# The 53 apps in the catalog on 2026-10-01 (catalog main 9c5eae9 + the checklist commit). They were published
# before the checklist existed; re-testing them is not owed (operator default 2026-10-01, may be reversed). What the
# catalog shows for each is onboarding/EXISTING-APPS-GAPS.md. NEVER add a name here to let a new app through.
EXEMPT = frozenset("""
actualbudget adventurelog audiobookshelf bentopdf bookstack calcom calibre-web claper code-server
crafty-controller docmost emby ghost gitea glance gokapi grafana gramps-web home-assistant homebox homepage
immich jellyfin kimai komga mealie n8n navidrome nextcloud onlyoffice opengist outline paperless-ngx papra
plant-it plex privatebin radarr rallly recipe-importer romm seerr sonarr sparkyfitness tandoor termix
uptime-kuma vaultwarden vikunja wanderer wger wishlist zipline
""".split())

ROW_CHECKLIST = re.compile(r"^\|\s*(\d+\.\d+)\s*\|\s*(\d{4}-\d{2}-\d{2})\s*\|")
ROW_RECORD = re.compile(r"^(\d+\.\d+)\s*\|\s*([^|]*?)\s*\|\s*(.*?)\s*$")
DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")


def read_checklist(root):
    path = os.path.join(root, "NEW-APP-CHECKLIST.md")
    if not os.path.isfile(path):
        return None, "NEW-APP-CHECKLIST.md is missing"
    ids = {}
    for line in open(path, encoding="utf-8"):
        m = ROW_CHECKLIST.match(line)
        if m:
            cid, since = m.group(1), m.group(2)
            if cid in ids:
                return None, "checklist id %s appears twice" % cid
            try:
                datetime.date.fromisoformat(since)
            except ValueError:
                return None, "checklist id %s has a bad since date %r" % (cid, since)
            ids[cid] = since
    if not ids:
        return None, "no `| <id> | <since> |` rows found in NEW-APP-CHECKLIST.md"
    return ids, None


def evidence_ok(p):
    if os.path.isfile(p):
        return os.path.getsize(p) > 0
    if os.path.isdir(p):
        for dp, _dn, fn in os.walk(p):
            for f in fn:
                if os.path.getsize(os.path.join(dp, f)) > 0:
                    return True
    return False


def resolve(root, rel, unchecked):
    """Return (abs path or None, problem or None). None, None = not checkable here (counted)."""
    rel = rel.strip().strip("`")
    if not rel or rel.startswith("/") or ".." in rel.split("/"):
        return None, "evidence %r is not a workspace-relative path" % rel
    if rel.startswith(CATALOG_PREFIX):
        return os.path.join(root, rel[len(CATALOG_PREFIX):]), None
    first = rel.split("/", 1)[0]
    sib = os.path.join(os.path.dirname(root), first)
    if not os.path.isdir(sib):
        unchecked.append(rel)
        return None, None
    return os.path.join(os.path.dirname(root), rel), None


def check_record(root, app, path, checklist, strict, today, unchecked):
    probs = []
    # A row inside an HTML comment is not an answer: `<!-- … -->` is how a row is set aside, and a commented-out
    # `1.4 | done | …` must not count as done (the label without the fact, R-421). Line numbers are kept.
    text = re.sub(r"<!--.*?-->", lambda m: "\n" * m.group(0).count("\n"),
                  open(path, encoding="utf-8").read(), flags=re.S)
    head = {}
    for line in text.splitlines():
        m = re.match(r"^(app|opened):\s*(\S+)\s*$", line)
        if m and m.group(1) not in head:
            head[m.group(1)] = m.group(2)
    if head.get("app") != app:
        probs.append("its `app:` line reads %r, not %r" % (head.get("app"), app))
    opened = head.get("opened", "")
    if not DATE.match(opened):
        probs.append("no `opened: YYYY-MM-DD` line")
        opened = None
    else:
        try:
            datetime.date.fromisoformat(opened)
        except ValueError:
            probs.append("`opened: %s` is not a real date" % opened)
            opened = None
    if strict and opened:
        if opened < CUTOFF:
            probs.append("`opened: %s` is before the checklist existed (%s) — a new app cannot be opened earlier"
                         % (opened, CUTOFF))
        if opened > today:
            probs.append("`opened: %s` is in the future (today %s)" % (opened, today))

    seen = {}
    for n, line in enumerate(text.splitlines(), 1):
        m = ROW_RECORD.match(line)
        if not m:
            continue
        cid, status, rest = m.group(1), m.group(2).lower(), m.group(3)
        if cid not in checklist:
            probs.append("line %d: id %s is not in the checklist" % (n, cid))
            continue
        if cid in seen:
            probs.append("line %d: id %s answered twice (first on line %d)" % (n, cid, seen[cid]))
            continue
        seen[cid] = n
        if status not in STATUSES:
            probs.append("line %d: id %s has status %r — one of done / n/a / open" % (n, cid, status))
        elif status == "open":
            if strict:
                probs.append("id %s is OPEN: %s" % (cid, rest or "(no note)"))
        elif status == "n/a":
            if len(re.findall(r"[^\W\d_]{2,}", rest)) < MIN_REASON_WORDS:
                probs.append("id %s is n/a with no reason of %d+ words: %r" % (cid, MIN_REASON_WORDS, rest))
        else:  # done
            paths = rest.split(" — ", 1)[0]
            parts = [p for p in (x.strip() for x in paths.split(" ; ")) if p]
            if not parts:
                probs.append("id %s is done with no evidence path" % cid)
            for p in parts:
                ap, why = resolve(root, p, unchecked)
                if why:
                    probs.append("id %s: %s" % (cid, why))
                elif ap and not evidence_ok(ap):
                    probs.append("id %s is done but its evidence %s does not exist (or is empty)" % (cid, p))
    if strict and opened:
        missing = [c for c, s in sorted(checklist.items(), key=lambda kv: [int(x) for x in kv[0].split(".")])
                   if s <= opened and c not in seen]
        if missing:
            probs.append("missing id(s): %s" % ", ".join(missing))
    return probs


def main(argv):
    root = ROOT
    today = datetime.date.today().isoformat()
    for a in argv:
        if a.startswith("--root="):
            root = os.path.abspath(a.split("=", 1)[1])
        elif a.startswith("--today="):
            today = a.split("=", 1)[1]
        elif a in ("--all",):
            pass  # the runner passes --all through; every template is judged anyway
        elif not a.startswith("-"):
            pass  # app scope is not used: the rule is about the whole templates/ tree
        else:
            print("unknown option %s" % a)
            return 2
    checklist, err = read_checklist(root)
    if err:
        print("ONBOARDING GATE INCONCLUSIVE — %s" % err)
        return 2

    tdir = os.path.join(root, "templates")
    odir = os.path.join(root, "onboarding")
    apps = sorted(d for d in os.listdir(tdir) if os.path.isdir(os.path.join(tdir, d)))
    new = [a for a in apps if a not in EXEMPT]
    problems, unchecked = [], []

    tpl = os.path.join(odir, "_TEMPLATE.md")
    if not os.path.isfile(tpl):
        problems.append(("_TEMPLATE", ["onboarding/_TEMPLATE.md is missing"]))
    else:
        body = re.sub(r"<!--.*?-->", "", open(tpl, encoding="utf-8").read(), flags=re.S)
        have = {m.group(1) for m in (ROW_RECORD.match(l) for l in body.splitlines()) if m}
        lack = sorted(set(checklist) - have, key=lambda c: [int(x) for x in c.split(".")])
        if lack:
            problems.append(("_TEMPLATE", ["onboarding/_TEMPLATE.md lacks checklist id(s): %s" % ", ".join(lack)]))

    for app in new:
        rec = os.path.join(odir, app + ".md")
        if not os.path.isfile(rec):
            problems.append((app, ["NEW app with no onboarding record — copy onboarding/_TEMPLATE.md to "
                                   "onboarding/%s.md and answer every id (NEW-APP-CHECKLIST.md)" % app]))
            continue
        p = check_record(root, app, rec, checklist, True, today, unchecked)
        if p:
            problems.append((app, p))

    exempt_records = []
    if os.path.isdir(odir):
        for f in sorted(os.listdir(odir)):
            name = f[:-3] if f.endswith(".md") else None
            if not name or name.startswith("_") or name.isupper() or "-GAPS" in name.upper():
                continue
            if name not in apps:
                problems.append((name, ["onboarding/%s.md names no template directory" % f]))
            elif name in EXEMPT:
                exempt_records.append(name)
                p = check_record(root, name, os.path.join(odir, f), checklist, False, today, unchecked)
                if p:
                    problems.append((name, p))

    print("onboarding gate — %d checklist ids; %d template dirs: %d exempt (published before %s), %d new; "
          "%d exempt app(s) with a record (shape-checked): %s"
          % (len(checklist), len(apps), len(apps) - len(new), CUTOFF, len(new), len(exempt_records),
             ", ".join(exempt_records) or "none"))
    if unchecked:
        print("  NOT CHECKED here (sibling repository absent — the pre-push hook on DooPlex checks them): %d path(s)"
              % len(unchecked))
        for u in unchecked[:10]:
            print("    %s" % u)
    if problems:
        print("ONBOARDING GATE FAILED:")
        for app, ps in problems:
            for p in ps:
                print("  %s: %s" % (app, p))
        return 1
    print("onboarding gate OK")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))