admin
458cf4a4c2
- scans all templates + Go sources (291 files) for the double-encoding signature chars (Ã Â Ă ă ˘ ˇ) + strict-UTF-8 decode; allowlist ZERO by design; Python per the multibyte rule (this task exists partly because of a bash-era mojibake sweep) - byte-level sweep result: the committed source contains NO mojibake — the 'Tárhely'-class text on the live import page is the felhom-usb DRIVE-LABEL DATA (settings.json on the demo box; default-label pattern settings.go GenerateLabel), repaired via the label-edit UI in the live-validation step, not by code - red-proven: reintroduced 'Tárhely' into app_import.html → exit 1 naming file:line; reverted → 0
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""v0.126.0 mojibake gate — double-encoded UTF-8 can never enter the source again.
|
|
|
|
A bash-era in-place sweep once mangled Hungarian text by re-encoding already-encoded
|
|
UTF-8 (the "Tárhely"-class defect: á → á). Legitimate Hungarian UTF-8 NEVER decodes
|
|
to the signature characters below, so any hit is a defect. The allowlist is ZERO by
|
|
design — fix the text, don't allowlist it. (Windows grep false-negatives multibyte —
|
|
this gate is Python by the multibyte rule, like emoji_gate.)
|
|
|
|
Scans every template (web + setup) and every Go source file:
|
|
- the file must decode as strict UTF-8 (undecodable bytes = broken already), and
|
|
- the decoded text must contain none of the double-encoding signature characters:
|
|
à (U+00C3)  (U+00C2) Ă (U+0102) ă (U+0103) ˘ (U+02D8) ˇ (U+02C7)
|
|
|
|
Run from controller/: python scripts/mojibake_gate.py
|
|
Exit 1 on any hit.
|
|
"""
|
|
import io, os, sys
|
|
|
|
ROOTS = [
|
|
os.path.join("internal", "web", "templates"),
|
|
os.path.join("internal", "setup", "templates"),
|
|
"internal", # every .go string literal / comment
|
|
"cmd",
|
|
]
|
|
|
|
SIGNATURE = {
|
|
"Ã": "Ã (a-tilde — á/é/í/ó/ö/ő/ú/ü/ű double-encoded)",
|
|
"Â": "Â (a-circumflex — NBSP/degree-sign double-encoded)",
|
|
"Ă": "Ă (a-breve — cp1250 round-trip)",
|
|
"ă": "ă (a-breve lowercase — cp1250 round-trip)",
|
|
"˘": "˘ (breve — cp1250 round-trip)",
|
|
"ˇ": "ˇ (caron — cp1250 round-trip)",
|
|
}
|
|
|
|
EXTS = (".html", ".css", ".js", ".go")
|
|
|
|
|
|
def files():
|
|
seen = set()
|
|
for root in ROOTS:
|
|
if not os.path.isdir(root):
|
|
continue
|
|
for dirpath, dirnames, names in os.walk(root):
|
|
for n in names:
|
|
p = os.path.join(dirpath, n)
|
|
if p in seen or not n.endswith(EXTS):
|
|
continue
|
|
seen.add(p)
|
|
yield p
|
|
|
|
|
|
failures = []
|
|
count = 0
|
|
for path in files():
|
|
count += 1
|
|
raw = io.open(path, "rb").read()
|
|
try:
|
|
text = raw.decode("utf-8")
|
|
except UnicodeDecodeError as e:
|
|
failures.append("%s: not valid UTF-8 (%s)" % (path, e))
|
|
continue
|
|
for ln, line in enumerate(text.split("\n"), 1):
|
|
for ch, why in SIGNATURE.items():
|
|
if ch in line:
|
|
failures.append("%s:%d: mojibake signature %s in %r" % (path, ln, why, line.strip()[:120]))
|
|
|
|
if failures:
|
|
print("mojibake_gate: FAIL (allowlist is zero — fix the text)")
|
|
for f in failures:
|
|
print(" - " + f)
|
|
sys.exit(1)
|
|
print("mojibake_gate: OK (%d files clean — no double-encoding signatures)" % count)
|