""" * Version: 0.1.0.0.0 * Purposes: * This is a result-driven style English to Simplified Chinese translation tool which is better than the static dictionary only style, it blocks the wrong translation while still allows the translation flexibility for auto translator. * This is an intenral utilities specifically for processing English to Simplified Chinese translation for pingcap.pe.flash module. """ import re, csv, importlib_resources import pingcap.pe.flash.util as util import pingcap.pe.flash.util_translator as translator import pingcap.pe.flash.ai_editor as editor _linematch_replace_resultmatch_ci = [ e[1:] for e in [ *csv.reader( importlib_resources.open_text( "pingcap.pe.flash", "flash-en2zh-blocklist-dictionary-ci.csv" ) ) ] ] _linematch_replace_resultmatch_cs = [ e[1:] for e in [ *csv.reader( importlib_resources.open_text( "pingcap.pe.flash", "flash-en2zh-blocklist-dictionary-cs.csv" ) ) ] ] _errata_cs = [ e[1:] for e in [ *csv.reader( importlib_resources.open_text( "pingcap.pe.flash", "flash-en2zh-static-errata-cs.csv" ) ) ] ] def _line_lower_result_replace( line, result, line_match: str, result_match: str, result_replacement: str ) -> str: if line_match in line.lower() and result_match in result: return result.replace(result_match, result_replacement) else: return result def _line_lower_result_replace_multi( line, result, line_match_lower: str, results_match: list, result_replacement: str ) -> str: for r in results_match: result = _line_lower_result_replace( line, result, line_match_lower, r, result_replacement ) additional_result_match = [] additional_lower_result_match = line_match_lower.split(" ") # All lower case: additional_result_match += [" ".join(additional_lower_result_match)] # All title case: additional_result_match += [ " ".join(e.title() for e in additional_lower_result_match) ] if " " in line_match_lower: # Dash connected case: additional_line_match_lower = line_match_lower.replace(" ", "-") result = _line_lower_result_replace_multi( line, result, additional_line_match_lower, results_match, result_replacement ) # First word title case: additional_result_match += [ additional_lower_result_match[0].title() + " " + " ".join(additional_lower_result_match[1:]) ] for x in additional_result_match: result = _line_lower_result_replace( line, result, line_match_lower, x, result_replacement ) return result def _line_result_replace( line, result, line_match: str, result_match: str, result_replacement: str ) -> str: if line_match in line and result_match in result: return result.replace(result_match, result_replacement) else: return result def _result_replace(result, result_match: str, result_replacement: str) -> str: if result_match in result: return result.replace(result_match, result_replacement) else: return result def _translate_cache_en2zh(line: str) -> str: """Return the cached Markdown translated result for English Markdown to Simplified Chinese Markdown. Args: line (str): The English source Markdown line. Returns: str: The cached Markdown translation result. """ if ( line.strip("\n").strip() == "No speaker notes." or line.strip("\n").strip() == "[no notes]" ): return "[无内容]" if line.strip("\n").strip() == "## Overview": return "## 概述" if line.strip("\n").strip() == "## Notes": return "## 注意事项" if line.strip("\n").strip() == "## Duration": return "## 持续时间" if ( line.strip("\n").strip() == "## Solution Steps" or line.strip("\n").strip() == "## Steps" ): return "## 步骤" if line.strip("\n").strip() == "## Outline": return "## 大纲" if line.strip("\n").strip() == "## Domain Details": return "## 知识领域" if line.strip("\n").strip() == "# Objectives": return "# 章节目标" return None def pre_en2zh_auto_translation_md_prefix_aware(line_in: str) -> str: """ This routine currently is NOT in-use. Args: line_in (str): The English line to be translated. Returns: str: The English line refined for translating into Simplified Chinese. """ processed_line = None processed_line = line_in return processed_line def post_en2zh_auto_translation( line: str, auto_translated_line: str, mode="standard", md_file_path=None ) -> str: """ This routine will be called in a context WITHOUT the Markdown prefix such as "#, +, -, *", etc. :param line: The English source line. :param auto_translated_line: The translated line by auto translation service. :returns: The post processed result. """ result = auto_translated_line.strip("\n") print(util.speak_result("Begin Post Handling for Input: " + result)) # 1. Placeholder. print(util.speak_result("Post Phase 1 Completed: " + result)) # 2. Placeholder. print(util.speak_result("Post Phase 2 Completed: " + result)) # 4. Correct case sensitive translation result by a static dictionary. # Load `line_match_cs, replacement, result_match_cs` from CSV. for e in _linematch_replace_resultmatch_cs[1:]: if not type(e[2]) == str: for i in e[2]: result = _line_result_replace(line, result, e[0], i, e[1]) else: result = _line_result_replace(line, result, e[0], e[2], e[1]) print(util.speak_result("Post Phase 4 Completed: " + result)) # 5. Correct case insensitive translation result. # Load `line_match_ci, replacement, result_match_cs` from CSV. for e in _linematch_replace_resultmatch_ci[1:]: result = _line_lower_result_replace_multi( line, result, e[0], e[2].split(","), e[1] ) print(util.speak_result("Post Phase 5 Completed: " + result)) # 6. Correct complex translation result. Goal keeper solutions for those issues that can not be solved in step 5 completely. if "key-value pair" in line.lower(): result = result.replace("键值对", "Key-Value pair") if "key-value" in line.lower(): result = result.replace("键-值", "Key-Value") if ( "key" in line.lower() and "keyword" not in line.lower() and "point" not in line.lower() ): if "关键词" in result: result = result.replace("关键词", "KEY") if "密钥" in result: result = result.replace("密钥", "KEY") if "关键字" in result: result = result.replace("关键字", "KEY") if "钥匙" in result: result = result.replace("钥匙", "KEY") if "键" in result: result = result.replace("键", "KEY") if "tidb/tikv/pd" in line.lower(): result = re.sub("tidb/tikv/pd", "TiDB/TiKV/PD", result, flags=re.IGNORECASE) print(util.speak_result("Post Phase 6 Completed: " + result)) # 7. REMOVERS: # Remove trailing "。" for bullets if result.strip().startswith("+"): result = result.strip("。") if result.strip().startswith("#"): result = result.strip("。") # Remove the generated ** if "**" in result and "**" not in line: result = result.replace("**", "") # Remove the unwanted "(PD 一乘一)" at the end if re.match("^`[^`]+` *(PD 一乘一)[。]*$", result): result = result.replace("(PD 一乘一)", "") print(util.speak_result("Post Phase 7 Completed: " + result)) return result def post_en2zh_auto_translation_md_prefix_aware(in_line: str, source_line: str) -> str: """ This routine will be called in a context with the Markdown prefix such as "#, +, -, *", etc. """ result = in_line if result.strip().startswith("+"): result = result.strip("。") if result.strip().startswith("#"): result = result.strip("。") if result.strip().startswith("["): result = result.strip("。") if not "(" in source_line and not ")" in source_line: result = re.sub("\(.*\)", "", result) if not '"' in source_line: result = result.replace('"', "") return result def translate_en2zh_md_line( line: str, en_1gram_list: list, mode="standard", md4ppt=True, md_file_path=None ) -> str: """ Translate markdown line from EN to JA. Here goes the logic: 0. Translation cache. 1. Short-cut the line in case translation is not needed. 2. Pack the prefixed whitepsaces from the line, for example: `_____## Hello world` 3. Pack the prefixed `#`, `+`, `-`, `+`, `*`, `$numbering` (whitespace included) from the line. 4. Replace the inline codeline - `CODE` with placeholders. Translate the processed line. 5. Put the inline codeline back. 6. Post translation handling. 7-12. Unpack `#`, `+`, `-`, `+`, `*`, `$numbering` and prefixed whitspaces back to the line. """ # Source print(util.speak_result("SOURCE: [ " + line + " ]")) # Pre-process line = pre_en2zh_auto_translation_md_prefix_aware(line) print(util.speak_result("AFTER PRE-PROCESS MD-AWARE: [ " + line + " ]")) print(util.speak_result("BEFORE TRANSLATION: [ " + line + " ]")) # 0. Translation cache. cache_result = _translate_cache_en2zh(line) if cache_result != None: print(util.speak_result("Translate from cache: " + cache_result)) return cache_result # 1. Do nothing: Shortcut. if translator.is_skippable(line): for e in _errata_cs[1:]: line = _result_replace(line, e[0], e[1]) return line # Assign `line` to `compress_space_raw` to keep the line intact. compress_space_raw = line.strip("\n") # 2. Pack the prefixed whitespace prefix_whitespace_pack = translator.pack_prefix_whitespace(compress_space_raw) compress_space_raw = prefix_whitespace_pack["processed_line"] # 2.1 Markdown table processing if translator.is_table_row(line.strip("\n")): compress_space_raw = translator.translate_table_row( compress_space_raw, translate_en2zh_md_line, en_1gram_list ) print(util.speak_result("Table row translated: " + compress_space_raw)) else: # 3.1 Pack the prefixed hash prefix_hash_pack = translator.pack_prefix_hash(compress_space_raw) compress_space_raw = prefix_hash_pack["processed_line"] # 3.2 Pack the prefixed plus prefix_plus_pack = translator.pack_prefix_plus(compress_space_raw) compress_space_raw = prefix_plus_pack["processed_line"] # 3.3 Pack the prefixed minus prefix_minus_pack = translator.pack_prefix_minus(compress_space_raw) compress_space_raw = prefix_minus_pack["processed_line"] # 3.4 Pack the prefixed star prefix_star_pack = translator.pack_prefix_star(compress_space_raw) compress_space_raw = prefix_star_pack["processed_line"] # 3.5 Pack the prefixed number prefix_number_pack = translator.pack_prefix_number(compress_space_raw) compress_space_raw = prefix_number_pack["processed_line"] # 3.6 Do nothing: Shortcut. if not translator.is_skippable(compress_space_raw): # 4. Pack inline codeline inline_codeline_pack = translator.pack_inline_codeline(compress_space_raw) compress_space_raw = inline_codeline_pack["processed_line"] # Ready for translation before_translation = compress_space_raw.strip() # Translate translated_line = editor.translate_en2zh_by_openai(before_translation) print( util.speak_result( "BEFORE OPENAI TRANSLATION: [ " + before_translation + " ]" ) ) print( util.speak_result( "AFTER OPENAI TRANSLATION: [ " + translated_line + " ]" ) ) compress_space_raw = translated_line.strip() # 5. Inline code handling compress_space_raw = translator.unpack_inline_codeline( compress_space_raw, inline_codeline_pack ) # Cut-off the whitespaces if any compress_space_raw = compress_space_raw.strip() # 6. Post translate print( util.speak_result( "BEFORE POST-TRANSLATION: [ " + compress_space_raw + " ]" ) ) compress_space_raw = post_en2zh_auto_translation( before_translation, compress_space_raw, mode, md_file_path ) print( util.speak_result( "AFTER POST-TRANSLATION: [ " + compress_space_raw + " ]" ) ) # 7. Adding Hash title prefix back compress_space_raw = translator.unpack_prefix_hash( compress_space_raw, prefix_hash_pack, en_1gram_list, md4ppt ) # 8. Adding plus prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_plus_pack ) # 9. Adding minus prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_minus_pack ) # 10. Adding star prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_star_pack ) # 11. Adding number prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_number_pack ) # 12. Give back the prefix whitespace compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_whitespace_pack ) # 13. Correct the general results in all cases. for e in _errata_cs[1:]: compress_space_raw = _result_replace(compress_space_raw, e[0], e[1]) # Finally print(util.speak_result("AFTER TRANSLATION: [ " + compress_space_raw + " ]")) if md4ppt: compress_space_raw = post_en2zh_auto_translation_md_prefix_aware( compress_space_raw, line ) print( util.speak_result( "AFTER TRANSLATION MD-AWARE: [ " + compress_space_raw + " ]" ) ) return compress_space_raw