felhom-controller/controller/scripts/i18n_missing_gate.py

# -*- coding: utf-8 -*-
"""i18n_missing_gate.py -- the message bundles are complete, consistent and in the product's voice.

Run from controller/:  python3 scripts/i18n_missing_gate.py
Exit 0 clean · 1 convicted · 2 inconclusive (bundles or templates not found).

Design: felhom.eu/documentation/architecture/10-localisation.md. The Go tests pin the runtime
(parity, fallback, parameters, JS-context safety); this gate pins the SOURCE, before a build:

  1. KEYS EXIST. Every `{{T "key"}}` marker in any dashboard template names a key in hu.json.
     (The loader refuses an undefined key at startup; this names the key at push time instead.)
  2. NO ORPHANS. Every en.json key translates a hu.json key (".one"/".other" count for their base).
  3. THE ENGLISH GAP ONLY SHRINKS. The number of hu keys with no English is printed and compared with
     EN_MISSING_CEILING. More than the ceiling convicts (a slice converted copy and did not translate
     it); LESS also convicts, until the ceiling is lowered to match -- a ratchet that cannot be
     loosened by forgetting it. Raising it is a decision recorded in 10-localisation.md.
  4. VOICE, per language.
     en -- second person, plain: no "please", no "kindly" (the household is not being begged).
     hu -- the product speaks in „te". Formal („ön") forms already in the converted copy are known
     debt (R-516) and COUNTED against HU_FORMAL_CEILING with the same ratchet: a new formal form
     convicts; fixing one requires lowering the ceiling. The Hungarian copy itself is not changed by
     the release that introduced this gate (parity rule), which is why this is a ratchet and not a ban.

Hungarian is matched by ASCII-folded stems (the workspace rule for Hungarian search), with a
positive and a negative control printed on every run.
"""
import io
import json
import os
import re
import sys
import unicodedata

HERE = os.path.dirname(os.path.abspath(__file__))
CTRL = os.path.dirname(HERE)
LOCALES = os.path.join(CTRL, "internal", "i18n", "locales")
TEMPLATE_ROOTS = [os.path.join(CTRL, "internal", "web", "templates")]
MARKER = re.compile(r'\{\{\s*T\s+"([A-Za-z0-9_.\-]+)"\s*\}\}')

EN_MISSING_CEILING = 0
HU_FORMAL_CEILING = 18  # 6 -> 10 -> 12 -> 16 (slice 1) -> 18 (slice 2 release A): the count follows the
# conversion, it is not a judgement. Moving a Go literal into hu.json makes an „ön" form the gate could
# not see before VISIBLE to it -- the two that arrived are `flash.login.password_changed` („kérjük")
# and `alert.deadapp.group` („nézze meg"), both word-for-word what the Go code already said. Slice 2 may
# not reword a Hungarian sentence (parity, 10-localisation.md s1), so the ceiling rises with the
# measurement and R-516 still owns the words.

EN_FORBIDDEN = re.compile(r"\b(please|kindly)\b", re.I)

# Formal-address stems, ASCII-folded. Each is a verb form or phrase that only the „ön" register uses.
HU_FORMAL = [
    "olvassa be", "kerjuk", "vegye fel", "ujratelepiti az", "importalnia", "biztosan kikapcsolja",
    "ellenorizze", "csatlakoztassa", "adjon hozza", "nezze meg", "kattintson", "valassza ki",
]


def fold(s):
    s = unicodedata.normalize("NFKD", s)
    return "".join(c for c in s if not unicodedata.combining(c)).lower()


def load(lang):
    p = os.path.join(LOCALES, lang + ".json")
    if not os.path.exists(p):
        return None
    return json.load(io.open(p, encoding="utf-8"))


def templates():
    out = []
    for root in TEMPLATE_ROOTS:
        for dirpath, _dirs, names in os.walk(root):
            for n in sorted(names):
                if n.endswith(".html"):
                    out.append(os.path.join(dirpath, n))
    return sorted(out)


def main():
    hu, en = load("hu"), load("en")
    tpls = templates()
    if hu is None or en is None or not tpls:
        print("i18n gate INCONCLUSIVE: bundles or templates not found under %s" % CTRL)
        return 2

    # Controls for the ASCII-folded matcher.
    if "kerjuk" not in fold("Kérjük, vegye fel"):
        print("i18n gate INCONCLUSIVE: positive control failed (folding does not strip accents)")
        return 2
    if any(stem in fold("Csatlakoztasd újra a meghajtót") for stem in HU_FORMAL):
        print("i18n gate INCONCLUSIVE: negative control failed (an informal te-form sentence matched a formal stem)")
        return 2

    problems = []
    used = 0
    for path in tpls:
        rel = os.path.relpath(path, CTRL)
        for i, line in enumerate(io.open(path, encoding="utf-8"), 1):
            for m in MARKER.finditer(line):
                used += 1
                if m.group(1) not in hu:
                    problems.append("%s:%d marker key %r is not in hu.json" % (rel, i, m.group(1)))

    for k in en:
        base = re.sub(r"\.(one|other)$", "", k)
        if k not in hu and base not in hu:
            problems.append("en.json key %r has no Hungarian source" % k)
        if EN_FORBIDDEN.search(en[k]):
            problems.append("en.json %r says %r -- plain second person, no pleading" % (k, EN_FORBIDDEN.search(en[k]).group(0)))

    missing = [k for k in hu if k not in en and (k + ".other") not in en]
    formal = [(k, stem) for k in hu for stem in HU_FORMAL if stem in fold(hu[k])]

    print("i18n: %d markers in %d templates; hu %d keys, en %d keys" % (used, len(tpls), len(hu), len(en)))
    print("i18n: English missing %d (ceiling %d); Hungarian formal forms %d (ceiling %d, R-516)"
          % (len(missing), EN_MISSING_CEILING, len(formal), HU_FORMAL_CEILING))
    for k, stem in formal:
        print("      formal: %s (%s)" % (k, stem))

    if len(missing) > EN_MISSING_CEILING:
        problems.append("%d Hungarian keys have no English, ceiling is %d: %s"
                        % (len(missing), EN_MISSING_CEILING, ", ".join(missing[:10])))
    elif len(missing) < EN_MISSING_CEILING:
        problems.append("English gap shrank to %d -- lower EN_MISSING_CEILING from %d to match (the ratchet)"
                        % (len(missing), EN_MISSING_CEILING))
    if len(formal) > HU_FORMAL_CEILING:
        problems.append("%d formal (on-form) forms in hu.json, ceiling %d -- the product speaks in the te-form" % (len(formal), HU_FORMAL_CEILING))
    elif len(formal) < HU_FORMAL_CEILING:
        problems.append("formal forms fell to %d -- lower HU_FORMAL_CEILING from %d to match (the ratchet)"
                        % (len(formal), HU_FORMAL_CEILING))

    if problems:
        for p in problems:
            print("  " + p)
        print("I18N GATE FAILED: %d problem(s)" % len(problems))
        return 1
    print("i18n gate OK")
    return 0


if __name__ == "__main__":
    sys.exit(main())