# -*- coding: utf-8 -*- """v0.126.0 mojibake gate — double-encoded UTF-8 can never enter the source again. A bash-era in-place sweep once mangled Hungarian text by re-encoding already-encoded UTF-8 (the "Tárhely"-class defect: á → á). Legitimate Hungarian UTF-8 NEVER decodes to the signature characters below, so any hit is a defect. The allowlist is ZERO by design — fix the text, don't allowlist it. (Windows grep false-negatives multibyte — this gate is Python by the multibyte rule, like emoji_gate.) Scans every template (web + setup) and every Go source file: - the file must decode as strict UTF-8 (undecodable bytes = broken already), and - the decoded text must contain none of the double-encoding signature characters: à (U+00C3)  (U+00C2) Ă (U+0102) ă (U+0103) ˘ (U+02D8) ˇ (U+02C7) Run from controller/: python scripts/mojibake_gate.py Exit 1 on any hit. """ import io, os, sys ROOTS = [ os.path.join("internal", "web", "templates"), os.path.join("internal", "setup", "templates"), "internal", # every .go string literal / comment "cmd", ] SIGNATURE = { "Ã": "à (a-tilde — á/é/í/ó/ö/ő/ú/ü/ű double-encoded)", "Â": " (a-circumflex — NBSP/degree-sign double-encoded)", "Ă": "Ă (a-breve — cp1250 round-trip)", "ă": "ă (a-breve lowercase — cp1250 round-trip)", "˘": "˘ (breve — cp1250 round-trip)", "ˇ": "ˇ (caron — cp1250 round-trip)", } EXTS = (".html", ".css", ".js", ".go") def files(): seen = set() for root in ROOTS: if not os.path.isdir(root): continue for dirpath, dirnames, names in os.walk(root): for n in names: p = os.path.join(dirpath, n) if p in seen or not n.endswith(EXTS): continue seen.add(p) yield p failures = [] count = 0 for path in files(): count += 1 raw = io.open(path, "rb").read() try: text = raw.decode("utf-8") except UnicodeDecodeError as e: failures.append("%s: not valid UTF-8 (%s)" % (path, e)) continue for ln, line in enumerate(text.split("\n"), 1): for ch, why in SIGNATURE.items(): if ch in line: failures.append("%s:%d: mojibake signature %s in %r" % (path, ln, why, line.strip()[:120])) if failures: print("mojibake_gate: FAIL (allowlist is zero — fix the text)") for f in failures: print(" - " + f) sys.exit(1) print("mojibake_gate: OK (%d files clean — no double-encoding signatures)" % count)