felhom-controller/controller/scripts/i18n_go_parity.py

# -*- coding: utf-8 -*-
"""i18n_go_parity.py -- a Go-side message key may only carry text that ALREADY existed.

Run from controller/:  python3 scripts/i18n_go_parity.py
Exit 0 clean * 1 convicted * 2 inconclusive (base capture or bundles missing).

Design: felhom.eu/documentation/architecture/10-localisation.md s1 -- the Hungarian product must
render byte-for-byte the same before and after every slice. Slice 1 proved that for TEMPLATES by
rendering fixtures captured from unconverted templates. Slice 2 moves GO literals into the bundle,
and a rendered fixture cannot cover them all (a flash, an API error, a country name each need their
own request). This gate proves the same property structurally, at push time:

  Every key slice 2 introduced maps to a Go string literal that existed at the BASE COMMIT, byte
  for byte. A literal that was split, joined, reworded or re-punctuated on the way into hu.json
  fails here and names both sides.

Two files carry it:

  scripts/i18n_go_base.json  -- EVERY Go string literal at the base commit, captured ONCE with
                                `--capture` from a clean worktree of that commit. FROZEN: it is the
                                measurement, and regenerating it to make a conversion pass is the
                                one thing that would make this gate a wish. Slice 2's three
                                releases all measure against the same capture.
  scripts/i18n_go_keys.json  -- `key -> from`, written by the conversion. `from` is the base
                                literal the key replaced, or the ORDERED list of literals a
                                concatenation joined.

Checks:
  1. LISTED. Every hu.json key named by Go code (s.msg / msgN / Msgf / MsgErrorf / b.Msg / b.Plural
     with a literal key) appears in i18n_go_keys.json. Forgetting to list a converted key is the
     hole this closes.
  2. REAL. Every `from` literal appears in the base capture. A key cannot cite text nobody wrote.
  3. BYTE-EQUAL. A single-literal key: hu.json[key] == from, exactly. A joined key: hu.json[key]
     with its printf verbs removed == the `from` fragments concatenated in order, with their verbs
     removed too. Plural keys compare their `.one`/`.other`-less base form.

The Go literal walker is the inventory's (felhom.eu/scripts/i18n_inventory.py `go_literals`),
copied rather than imported: this gate runs from a controller clone that may not sit beside
felhom.eu, and a gate that can fail to import its own scanner is a gate that can silently skip.

This source is ASCII-only: it holds no Hungarian, so there is nothing for a locale to mangle.
"""
from __future__ import annotations

import argparse
import io
import json
import os
import re
import sys
import urllib.parse

HERE = os.path.dirname(os.path.abspath(__file__))
CTRL = os.path.dirname(HERE)
LOCALES = os.path.join(CTRL, "internal", "i18n", "locales")
BASE_FILE = os.path.join(HERE, "i18n_go_base.json")
KEYS_FILE = os.path.join(HERE, "i18n_go_keys.json")

# Directories whose Go literals are IN SCOPE for the capture. cmd/ and internal/ minus the
# first-boot wizard, which is out of scope and slated for deletion (R-554, 10-localisation.md s6).
SCOPE_ROOTS = ["internal", "cmd"]
SCOPE_SKIP = [os.path.join("internal", "setup")]

# A string literal SHAPED like a bundle key: lower-case dotted segments, no spaces. Every such
# literal that hu.json actually knows is treated as "this key is named in Go".
#
# It deliberately does NOT look for `s.msg(` and its siblings. The first version did, and a decoy
# caught it: a key reaches the bundle through many shapes that are not a call to a message helper --
# `offboxRedirect(w, r, "flash.offbox.app_missing", true)` puts one in a redirect URL, an Alert
# carries one in a struct field, a handler stores one in `data["TitleKey"]`. Every one of those was
# invisible, so a converted key could be dropped from the map and nothing would say so. Matching the
# SHAPE and then requiring the bundle to know it cannot miss a delivery mechanism, because it does
# not model one.
KEY_SHAPE_RE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z0-9_\-]+)+$")

# printf verbs, including explicit argument indexes (%[2]s -- how English reorders while the
# Hungarian format string keeps the plain verbs it always had).
VERB_RE = re.compile(r"%(?:\[\d+\])?[-+# 0-9.*]*[a-zA-Z]")


def go_literals(src: str):
    """Yield (line, literal_body, raw) for every string literal outside comments.

    Copied from felhom.eu/scripts/i18n_inventory.py `go_literals` (2026-09-17).
    """
    i, n, line = 0, len(src), 1
    while i < n:
        c = src[i]
        if c == "\n":
            line += 1
            i += 1
        elif src.startswith("//", i):
            j = src.find("\n", i)
            i = n if j < 0 else j
        elif src.startswith("/*", i):
            j = src.find("*/", i + 2)
            j = n if j < 0 else j + 2
            line += src.count("\n", i, j)
            i = j
        elif c == "'":
            j = i + 1
            while j < n and src[j] != "'":
                j += 2 if src[j] == "\\" else 1
            i = j + 1
        elif c == '"':
            j = i + 1
            while j < n and src[j] != '"' and src[j] != "\n":
                j += 2 if src[j] == "\\" else 1
            yield line, src[i + 1:j], False
            i = j + 1
        elif c == "`":
            j = src.find("`", i + 1)
            j = n if j < 0 else j
            yield line, src[i + 1:j], True
            line += src.count("\n", i, j)
            i = j + 1
        else:
            i += 1


def go_files(root: str):
    out = []
    for d in SCOPE_ROOTS:
        base = os.path.join(root, d)
        if not os.path.isdir(base):
            continue
        for dirpath, _dirs, names in os.walk(base):
            rel = os.path.relpath(dirpath, root)
            if any(rel == s or rel.startswith(s + os.sep) for s in SCOPE_SKIP):
                continue
            for n in sorted(names):
                if n.endswith(".go") and not n.endswith("_test.go"):
                    out.append(os.path.join(dirpath, n))
    return sorted(out)


def read(p: str) -> str:
    with io.open(p, encoding="utf-8", errors="replace") as fh:
        return fh.read()


def unescape(body: str) -> str:
    """Turn a Go interpreted-literal BODY into the runtime string.

    Only the escapes this codebase uses appear here; anything unknown is left alone rather than
    guessed, which keeps the comparison honest (an unrecognised escape can only make a key FAIL
    to match, never falsely match).
    """
    out, i, n = [], 0, len(body)
    simple = {"n": "\n", "t": "\t", "r": "\r", "\\": "\\", '"': '"', "'": "'"}
    while i < n:
        if body[i] == "\\" and i + 1 < n:
            c = body[i + 1]
            if c in simple:
                out.append(simple[c])
                i += 2
                continue
            if c == "u" and i + 6 <= n:
                try:
                    out.append(chr(int(body[i + 2:i + 6], 16)))
                    i += 6
                    continue
                except ValueError:
                    pass
        out.append(body[i])
        i += 1
    return "".join(out)


def capture(root: str) -> dict:
    """Index EVERY Go string literal at this commit, not only the ones a word list calls Hungarian.

    The first version filtered by `has_hu`, and the filter was wrong in exactly the way R-565 named:
    a Hungarian string spelled without accents ("Naponta", "5 percenkent", "Eletjel (Heartbeat)",
    "Adatbazis mentes") is invisible to a letter search, and a word list is a list -- it is short by
    construction and nobody knows which words are missing. Seven such literals were silently absent
    from the first capture and the gate correctly refused the keys that cited them.

    Filtering is not needed here at all. This index answers ONE question -- "did the base commit
    contain this exact text?" -- and an unfiltered index answers it for every string. It cannot be
    fooled by an English literal that happens to equal a Hungarian sentence, because no such literal
    exists. So the word list is gone and the blind spot with it.
    """
    lits = {}
    for p in go_files(root):
        rel = os.path.relpath(p, root)
        src = read(p)
        for line, body, raw in go_literals(src):
            text = body if raw else unescape(body)
            if text == "":
                continue
            lits.setdefault(text, []).append("%s:%d" % (rel, line))
    return lits


def strip_verbs(s: str) -> str:
    """Remove printf verbs, keeping a literal %% as a single percent sign."""
    s = s.replace("%%", "\x00")
    s = VERB_RE.sub("", s)
    return s.replace("\x00", "%")


def load_json(p: str):
    if not os.path.exists(p):
        return None
    with io.open(p, encoding="utf-8") as fh:
        return json.load(fh)


def keys_named_in_go(root: str, hu):
    """Every bundle key a Go literal names -> the sites that name it."""
    out = {}
    for p in go_files(root):
        rel = os.path.relpath(p, root)
        src = read(p)
        for i, line in enumerate(src.splitlines(), 1):
            if line.lstrip().startswith("//"):
                continue
            for _line, body, raw in go_literals(line):
                text = body if raw else unescape(body)
                if hu is not None and text in hu and KEY_SHAPE_RE.match(text):
                    out.setdefault(text, []).append("%s:%d" % (rel, i))
    return out


def base_form(hu: dict, key: str):
    """The Hungarian text for key, accepting the plural split (key, key.one, key.other)."""
    if key in hu:
        return hu[key]
    for suffix in (".other", ".one"):
        if key + suffix in hu:
            return hu[key + suffix]
    return None


def main(argv) -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--capture", action="store_true",
                    help="rewrite i18n_go_base.json (run once, from a CLEAN worktree of the base commit)")
    ap.add_argument("--root", default=CTRL,
                    help="controller root to capture FROM -- point it at a clean worktree of the base "
                         "commit, never at a tree that has already been converted")
    ap.add_argument("--commit", default="", help="the commit being captured, recorded in the file")
    args = ap.parse_args(argv[1:])

    if args.capture:
        lits = capture(args.root)
        payload = {
            "_comment": (
                "FROZEN measurement -- every Hungarian Go string literal at the base commit of "
                "localisation slice 2. Regenerating this to make a conversion pass is the one thing "
                "that would turn i18n_go_parity.py into a wish. See its docstring."
            ),
            "commit": args.commit,
            "count": len(lits),
            # One site per literal: enough to name where the text came from in a failure, and it
            # keeps a frozen measurement small enough to read in a review.
            "literals": {k: sorted(set(v))[0] for k, v in sorted(lits.items())},
        }
        with io.open(BASE_FILE, "w", encoding="utf-8") as fh:
            json.dump(payload, fh, ensure_ascii=False, indent=1, sort_keys=False)
            fh.write("\n")
        print("captured %d Go string literals from %s -> %s"
              % (len(lits), args.root, os.path.relpath(BASE_FILE, CTRL)))
        return 0

    base = load_json(BASE_FILE)
    keys = load_json(KEYS_FILE)
    hu = load_json(os.path.join(LOCALES, "hu.json"))
    if base is None or keys is None or hu is None:
        print("go-parity gate INCONCLUSIVE: base capture, key map or hu.json not found under %s" % CTRL)
        return 2

    base_lits = base.get("literals", {})
    named = keys_named_in_go(CTRL, hu)
    # Keys that predate slice 2 (the spike's and slice 1's copy-producing funcs). Each is pinned by
    # its own Go test, named in the map; this gate accounts for them so a NEW Go key cannot hide
    # among them. They are not measured against the base capture: their Hungarian was moved by
    # slice 0/1 and proved by rendered fixtures, which is the stronger measurement.
    preexisting = keys.pop("_preexisting", {}) or {}
    keys.pop("_comment", None)
    bad = []

    # 1. LISTED -- a key a Go call site names must be accounted for.
    for key in sorted(named):
        if key not in keys and key not in preexisting:
            bad.append("UNLISTED  %s  named at %s but absent from %s"
                       % (key, named[key][0], os.path.basename(KEYS_FILE)))

    # 2 + 3. REAL and BYTE-EQUAL.
    for key in sorted(keys):
        spec = keys[key]
        value = base_form(hu, key)
        if value is None:
            bad.append("NO-KEY    %s  listed but absent from hu.json" % key)
            continue

        # A third spec shape, for the handful of sites where the Hungarian was written ALREADY
        # URL-ESCAPED inside a redirect target: {"inside": "<the base literal>"}. The literal is
        # unescaped and the key's text must occur in it exactly once. Explicit and rare on purpose --
        # a substring rule applied everywhere would let a reworded sentence pass.
        if isinstance(spec, dict):
            lit = spec.get("inside", "")
            if lit not in base_lits:
                bad.append("INVENTED  %s  cites a base literal that does not exist: %r" % (key, lit))
                continue
            decoded = urllib.parse.unquote_plus(lit)
            n = decoded.count(value)
            if n != 1:
                bad.append("CHANGED   %s  occurs %d times (want 1) in the unescaped literal\n"
                           "            hu.json  : %r\n"
                           "            unescaped: %r" % (key, n, value, decoded))
            continue

        froms = [spec] if isinstance(spec, str) else list(spec)
        missing = [f for f in froms if f not in base_lits]
        if missing:
            bad.append("INVENTED  %s  cites text that is in no base-commit literal: %r"
                       % (key, missing[0]))
            continue
        # A STRING `from` means "this key replaced exactly that literal" -- compared byte for byte, so
        # even a verb swapped from %s to %v convicts. A LIST means "this key joined these literals",
        # which is how a concatenation or a trailing parameter becomes one message; there the printf
        # verbs are what the join replaced, so both sides are compared with the verbs removed.
        if isinstance(spec, str):
            if value != froms[0]:
                bad.append("CHANGED   %s\n            hu.json: %r\n            base   : %r  (%s)"
                           % (key, value, froms[0], base_lits[froms[0]]))
        else:
            got, want = strip_verbs(value), strip_verbs("".join(froms))
            if got != want:
                bad.append("CHANGED   %s  (joined from %d literals)\n"
                           "            hu.json (verbs removed): %r\n"
                           "            base    (verbs removed): %r"
                           % (key, len(froms), got, want))

    print("go-parity: base capture %s (%d literals), %d slice-2 keys listed, %d pre-existing, "
          "%d named in Go"
          % (base.get("commit", "?")[:12] or "?", len(base_lits), len(keys), len(preexisting),
             len(named)))
    if bad:
        print("\ngo-parity gate CONVICTS (%d):" % len(bad))
        for b in bad:
            print("  " + b)
        return 1
    print("go-parity gate OK: every Go-side key carries base-commit text, byte for byte.")
    return 0


if __name__ == "__main__":
    sys.exit(main(sys.argv))