diff --git a/controller/scripts/mojibake_gate.py b/controller/scripts/mojibake_gate.py new file mode 100644 index 0000000..e64c64e --- /dev/null +++ b/controller/scripts/mojibake_gate.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +"""v0.126.0 mojibake gate — double-encoded UTF-8 can never enter the source again. + +A bash-era in-place sweep once mangled Hungarian text by re-encoding already-encoded +UTF-8 (the "Tárhely"-class defect: á → á). Legitimate Hungarian UTF-8 NEVER decodes +to the signature characters below, so any hit is a defect. The allowlist is ZERO by +design — fix the text, don't allowlist it. (Windows grep false-negatives multibyte — +this gate is Python by the multibyte rule, like emoji_gate.) + +Scans every template (web + setup) and every Go source file: + - the file must decode as strict UTF-8 (undecodable bytes = broken already), and + - the decoded text must contain none of the double-encoding signature characters: + à (U+00C3)  (U+00C2) Ă (U+0102) ă (U+0103) ˘ (U+02D8) ˇ (U+02C7) + +Run from controller/: python scripts/mojibake_gate.py +Exit 1 on any hit. +""" +import io, os, sys + +ROOTS = [ + os.path.join("internal", "web", "templates"), + os.path.join("internal", "setup", "templates"), + "internal", # every .go string literal / comment + "cmd", +] + +SIGNATURE = { + "Ã": "à (a-tilde — á/é/í/ó/ö/ő/ú/ü/ű double-encoded)", + "Â": " (a-circumflex — NBSP/degree-sign double-encoded)", + "Ă": "Ă (a-breve — cp1250 round-trip)", + "ă": "ă (a-breve lowercase — cp1250 round-trip)", + "˘": "˘ (breve — cp1250 round-trip)", + "ˇ": "ˇ (caron — cp1250 round-trip)", +} + +EXTS = (".html", ".css", ".js", ".go") + + +def files(): + seen = set() + for root in ROOTS: + if not os.path.isdir(root): + continue + for dirpath, dirnames, names in os.walk(root): + for n in names: + p = os.path.join(dirpath, n) + if p in seen or not n.endswith(EXTS): + continue + seen.add(p) + yield p + + +failures = [] +count = 0 +for path in files(): + count += 1 + raw = io.open(path, "rb").read() + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as e: + failures.append("%s: not valid UTF-8 (%s)" % (path, e)) + continue + for ln, line in enumerate(text.split("\n"), 1): + for ch, why in SIGNATURE.items(): + if ch in line: + failures.append("%s:%d: mojibake signature %s in %r" % (path, ln, why, line.strip()[:120])) + +if failures: + print("mojibake_gate: FAIL (allowlist is zero — fix the text)") + for f in failures: + print(" - " + f) + sys.exit(1) +print("mojibake_gate: OK (%d files clean — no double-encoding signatures)" % count)