lang-check.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332
  1. #!/usr/bin/env python3
  2. #
  3. # Version 1.0.2 - Build 43
  4. #############################################################################
  5. # Change log:
  6. # 7 May 2019, ondratu , Initial
  7. # 13 June 2019, 3d-gussner, Fix length false positives
  8. # 14 Sep. 2019, 3d-gussner, Prepare adding new language
  9. # 18 Sep. 2020, 3d-gussner, Fix execution of lang-check.py
  10. # 2 Apr. 2021, 3d-gussner, Fix and improve text warp
  11. # 22 Apr. 2021, DRracer , add English source to output
  12. # 23 Apr. 2021, wavexx , improve
  13. # 24 Apr. 2021, wavexx , improve
  14. # 26 Apr. 2021, wavexx , add character ruler
  15. # 21 Dec. 2021, 3d-gussner, Prepare more community languages
  16. # Swedish
  17. # Danish
  18. # Slovanian
  19. # Hungarian
  20. # Luxembourgian
  21. # Croatian
  22. # 3 Jan. 2022, 3d-gussner, Prepare Lithuanian
  23. # 7 Jan. 2022, 3d-gussner, Check for Syntax errors and exit with error
  24. # , add Build number 'git rev-list --count HEAD lang-check.py'
  25. # 30 Jan. 2022, 3d-gussner, Add arguments. Requested by @AttilaSVK
  26. # --information == output all source and translated messages
  27. # --import-check == used by `lang-import.sh`to verify
  28. # newly import `lang_en_??.txt` files
  29. # 14 Mar. 2022, 3d-gussner, Check if translation isn't equal to origin
  30. #############################################################################
  31. """Check PO files for formatting errors."""
  32. from argparse import ArgumentParser
  33. from sys import stdout, stderr, exit
  34. import codecs
  35. import polib
  36. import textwrap
  37. import re
  38. import os
  39. from lib import charset as cs
  40. from lib.io import load_map
  41. COLORIZE = (stdout.isatty() and os.getenv("TERM", "dumb") != "dumb") or os.getenv('NO_COLOR') == "0"
  42. def color_maybe(color_attr, text):
  43. if COLORIZE:
  44. return '\033[0;' + str(color_attr) + 'm' + text + '\033[0m'
  45. else:
  46. return text
  47. red = lambda text: color_maybe(31, text)
  48. green = lambda text: color_maybe(32, text)
  49. yellow = lambda text: color_maybe(33, text)
  50. cyan = lambda text: color_maybe(36, text)
  51. def print_wrapped(wrapped_text, rows, cols):
  52. if type(wrapped_text) == str:
  53. wrapped_text = [wrapped_text]
  54. for r, line in enumerate(wrapped_text):
  55. r_ = str(r + 1).rjust(3)
  56. if r >= rows:
  57. r_ = red(r_)
  58. print((' {} |{:' + str(cols) + 's}|').format(r_, line))
  59. def print_truncated(text, cols):
  60. if len(text) <= cols:
  61. prefix = text.ljust(cols)
  62. suffix = ''
  63. else:
  64. prefix = text[0:cols]
  65. suffix = red(text[cols:])
  66. print(' |' + prefix + '|' + suffix)
  67. def print_ruler(spc, cols):
  68. print(' ' * spc + cyan(('₀₁₂₃₄₅₆₇₈₉'*4)[:cols]))
  69. def print_source_translation(source, translation, wrapped_source, wrapped_translation, rows, cols):
  70. if rows == 1:
  71. print(' source text:')
  72. print_ruler(4, cols);
  73. print_truncated(source, cols)
  74. print(' translated text:')
  75. print_ruler(4, cols);
  76. print_truncated(translation, cols)
  77. else:
  78. print(' source text:')
  79. print_ruler(6, cols);
  80. print_wrapped(wrapped_source, rows, cols)
  81. print(' translated text:')
  82. print_ruler(6, cols);
  83. print_wrapped(wrapped_translation, rows, cols)
  84. print()
  85. def highlight_trailing_white(text):
  86. if type(text) == str:
  87. return re.sub(r' $', '·', text)
  88. else:
  89. ret = text[:]
  90. ret[-1] = highlight_trailing_white(ret[-1])
  91. return ret
  92. def wrap_text(text, cols):
  93. ret = []
  94. for line in text.split('\n'):
  95. # wrap each input line in text individually
  96. tmp = list(textwrap.TextWrapper(width=cols).wrap(line))
  97. if len(ret):
  98. # add back trailing whitespace
  99. tmp[-1] += ' ' * (len(text) - len(text.rstrip()))
  100. ret.extend(tmp)
  101. return ret
  102. def ign_char_first(c):
  103. return c.isalnum() or c in {'%', '?'}
  104. def ign_char_last(c):
  105. return c.isalnum() or c in {'.', "'"}
  106. def check_translation(entry, msgids, is_pot, no_warning, no_suggest, warn_empty, warn_same, information):
  107. """Check strings to display definition."""
  108. # fetch/decode entry for easy access
  109. meta = entry.comment.split('\n', 1)[0]
  110. source = entry.msgid
  111. translation = entry.msgstr
  112. line = entry.linenum
  113. known_msgid = msgids is None or source in msgids
  114. errors = 0
  115. # Check comment syntax (non-empty and include a MSG id)
  116. if known_msgid or warn_empty:
  117. if len(meta) == 0:
  118. print(red("[E]: Translation doesn't contain any comment metadata on line %d" % line))
  119. return False
  120. if not meta.startswith('MSG'):
  121. print(red("[E]: Critical syntax error: comment doesn't start with MSG on line %d" % line))
  122. print(red(" comment: " + meta))
  123. return False
  124. # Check if columns and rows are defined
  125. tokens = meta.split(' ')
  126. cols = None
  127. rows = None
  128. for item in tokens[1:]:
  129. try:
  130. key, val = item.split('=')
  131. if key == 'c':
  132. cols = int(val)
  133. elif key == 'r':
  134. rows = int(val)
  135. else:
  136. raise ValueError
  137. except ValueError:
  138. print(red("[E]: Invalid display definition on line %d" % line))
  139. print(red(" definition: " + meta))
  140. return False
  141. if cols is None and rows is None:
  142. if not no_warning and known_msgid:
  143. errors += 1
  144. print(yellow("[W]: No usable display definition on line %d" % line))
  145. # probably fullscreen, guess from the message length to continue checking
  146. cols = len(source)
  147. if rows is None:
  148. rows = 1
  149. elif rows > 1 and cols != 20:
  150. errors += 1
  151. print(yellow("[W]: Multiple rows with odd number of columns on line %d" % line))
  152. # Check if translation contains unsupported characters
  153. invalid_char = cs.translation_check(cs.unicode_to_source(translation))
  154. if invalid_char is not None:
  155. print(red('[E]: Critical syntax: Unhandled char %s found on line %d' % (repr(invalid_char), line)))
  156. print(red(' translation: ' + translation))
  157. return False
  158. # Pre-process the translation to translated characters for a correct preview and length check
  159. translation = cs.trans_replace(translation)
  160. wrapped_source = wrap_text(source, cols)
  161. rows_count_source = len(wrapped_source)
  162. wrapped_translation = wrap_text(translation, cols)
  163. rows_count_translation = len(wrapped_translation)
  164. # Incorrect number of rows/cols on the definition
  165. if rows == 1 and (len(source) > cols or rows_count_source > rows):
  166. errors += 1
  167. print(yellow('[W]: Source text longer than %d cols as defined on line %d:' % (cols, line)))
  168. print_ruler(4, cols);
  169. print_truncated(source, cols)
  170. print()
  171. elif rows_count_source > rows:
  172. errors += 1
  173. print(yellow('[W]: Wrapped source text longer than %d rows as defined on line %d:' % (rows, line)))
  174. print_ruler(6, cols);
  175. print_wrapped(wrapped_source, rows, cols)
  176. print()
  177. # All further checks are against the translation
  178. if is_pot:
  179. return (errors == 0)
  180. # Missing translation
  181. if len(translation) == 0 and (known_msgid or warn_empty):
  182. errors += 1
  183. if rows == 1:
  184. print(yellow("[W]: Empty translation for \"%s\" on line %d" % (source, line)))
  185. else:
  186. print(yellow("[W]: Empty translation on line %d" % line))
  187. print_ruler(6, cols);
  188. print_wrapped(wrapped_source, rows, cols)
  189. print()
  190. # Check for translation lenght
  191. if (rows_count_translation > rows) or (rows == 1 and len(translation) > cols):
  192. errors += 1
  193. print(red('[E]: Text is longer than definition on line %d: cols=%d rows=%d (rows diff=%d)'
  194. % (line, cols, rows, rows_count_translation-rows)))
  195. print_source_translation(source, translation,
  196. wrapped_source, wrapped_translation,
  197. rows, cols)
  198. # Different count of % sequences
  199. if source.count('%') != translation.count('%') and len(translation) > 0:
  200. errors += 1
  201. print(red('[E]: Unequal count of %% escapes on line %d:' % (line)))
  202. print_source_translation(source, translation,
  203. wrapped_source, wrapped_translation,
  204. rows, cols)
  205. # Different first/last character
  206. if not no_suggest and len(source) > 0 and len(translation) > 0:
  207. source_end = source.rstrip()[-1]
  208. translation_end = translation.rstrip()[-1]
  209. start_diff = not (ign_char_first(source[0]) and ign_char_first(translation[0])) and source[0] != translation[0]
  210. end_diff = not (ign_char_last(source_end) and ign_char_last(translation_end)) and source_end != translation_end
  211. if start_diff or end_diff:
  212. if start_diff:
  213. print(yellow('[S]: Differing first punctuation character (%s => %s) on line %d:' % (source[0], translation[0], line)))
  214. if end_diff:
  215. print(yellow('[S]: Differing last punctuation character (%s => %s) on line %d:' % (source[-1], translation[-1], line)))
  216. print_source_translation(source, translation,
  217. wrapped_source, wrapped_translation,
  218. rows, cols)
  219. if not no_suggest and source == translation and (warn_same or len(source.split(' ', 1)) > 1):
  220. print(yellow('[S]: Translation same as original on line %d:' %line))
  221. print_source_translation(source, translation,
  222. wrapped_source, wrapped_translation,
  223. rows, cols)
  224. # Short translation
  225. if not no_suggest and len(source) > 0 and len(translation) > 0:
  226. if len(translation.rstrip()) < len(source.rstrip()) / 2:
  227. print(yellow('[S]: Short translation on line %d:' % (line)))
  228. print_source_translation(source, translation,
  229. wrapped_source, wrapped_translation,
  230. rows, cols)
  231. # Incorrect trailing whitespace in translation
  232. if not no_warning and len(translation) > 0 and \
  233. (source.rstrip() == source or (rows == 1 and len(source) == cols)) and \
  234. translation.rstrip() != translation and \
  235. (rows > 1 or len(translation) != len(source)):
  236. errors += 1
  237. print(yellow('[W]: Incorrect trailing whitespace for translation on line %d:' % (line)))
  238. source = highlight_trailing_white(source)
  239. translation = highlight_trailing_white(translation)
  240. wrapped_translation = highlight_trailing_white(wrapped_translation)
  241. print_source_translation(source, translation,
  242. wrapped_source, wrapped_translation,
  243. rows, cols)
  244. # show the information
  245. if information and errors == 0:
  246. print(green('[I]: %s' % (meta)))
  247. print_source_translation(source, translation,
  248. wrapped_source, wrapped_translation,
  249. rows, cols)
  250. return (errors == 0)
  251. def main():
  252. """Main function."""
  253. parser = ArgumentParser(description=__doc__)
  254. parser.add_argument("po", help="PO file to check")
  255. parser.add_argument(
  256. "--no-warning", action="store_true",
  257. help="Disable warnings")
  258. parser.add_argument(
  259. "--no-suggest", action="store_true",
  260. help="Disable suggestions")
  261. parser.add_argument(
  262. "--pot", action="store_true",
  263. help="Do not check translations")
  264. parser.add_argument(
  265. "--information", action="store_true",
  266. help="Output all translations")
  267. parser.add_argument("--map",
  268. help="Provide a map file to suppress warnings about unused translations")
  269. parser.add_argument(
  270. "--warn-empty", action="store_true",
  271. help="Warn about empty definitions and translations even if unused")
  272. parser.add_argument(
  273. "--warn-same", action="store_true",
  274. help="Warn about one-word translations which are identical to the source")
  275. # load the translations
  276. args = parser.parse_args()
  277. if not os.path.isfile(args.po):
  278. print("{}: file does not exist or is not a regular file".format(args.po), file=stderr)
  279. return 1
  280. # load the symbol map to supress empty (but unused) translation warnings
  281. msgids = None
  282. if args.map:
  283. msgids = set()
  284. for sym in load_map(args.map):
  285. if type(sym['data']) == bytes:
  286. msgid = cs.source_to_unicode(codecs.decode(sym['data'], 'unicode_escape', 'strict'))
  287. msgids.add(msgid)
  288. # check each translation in turn
  289. status = True
  290. for translation in polib.pofile(args.po):
  291. status &= check_translation(translation, msgids, args.pot, args.no_warning, args.no_suggest,
  292. args.warn_empty, args.warn_same, args.information)
  293. return 0 if status else os.EX_DATAERR
  294. if __name__ == "__main__":
  295. exit(main())