""" * Version: 0.1.0.0.0 * Purposes: * This is a result-driven style English to Japanese translation tool which is better than static dictionary style, it blocks the wrong translation while still allows the translation flexibility for auto translator. * This is an intenral utilities specifically for processing English to Japanese translation for pingcap.pe.flash module. * This tool prefers "ですます体", it's working in progress, always. """ import re, csv, importlib_resources import pingcap.pe.flash.util as util import pingcap.pe.flash.util_translator as translator import pingcap.pe.flash.ai_editor as editor _linematch_replace_resultmatch_ci = [ e[1:] for e in [ *csv.reader( importlib_resources.open_text( "pingcap.pe.flash", "flash-en2ja-blocklist-dictionary-ci.csv" ) ) ] ] _linematch_replace_resultmatch_cs = [ e[1:] for e in [ *csv.reader( importlib_resources.open_text( "pingcap.pe.flash", "flash-en2ja-blocklist-dictionary-cs.csv" ) ) ] ] _errata_cs = [ e[1:] for e in [ *csv.reader( importlib_resources.open_text( "pingcap.pe.flash", "flash-en2ja-static-errata-cs.csv" ) ) ] ] _a_column = { "あ": "ア", "か": "カ", "さ": "サ", "た": "タ", "な": "ナ", "は": "ハ", "ま": "マ", "や": "ヤ", "ら": "ラ", "わ": "ワ", "が": "ガ", "ざ": "ザ", "だ": "ダ", "ば": "バ", "ぱ": "パ", } _i_column = { "い": "イ", "き": "キ", "し": "シ", "ち": "チ", "に": "ニ", "ひ": "ヒ", "み": "ミ", "り": "リ", "ぎ": "ギ", "じ": "ジ", "ぢ": "ヂ", "び": "ビ", "ぴ": "ピ", } _u_column = { "う": "ウ", "く": "ク", "す": "ス", "つ": "ツ", "ぬ": "ヌ", "ふ": "フ", "む": "ム", "ゆ": "ユ", "る": "ル", "ぐ": "グ", "ず": "ズ", "づ": "ヅ", "ぶ": "ブ", "ぷ": "プ", } _e_column = { "え": "エ", "け": "ケ", "せ": "セ", "て": "テ", "ね": "ネ", "へ": "ヘ", "め": "メ", "れ": "レ", "げ": "ゲ", "ぜ": "ゼ", "で": "デ", "べ": "ベ", "ぺ": "ペ", } _o_column = { "お": "オ", "こ": "コ", "そ": "ソ", "と": "ト", "の": "ノ", "ひ": "ヒ", "も": "モ", "よ": "ヨ", "ろ": "ロ", "を": "ヲ", "ん": "ン", "ご": "ゴ", "ぞ": "ゾ", "ど": "ド", "ぼ": "ボ", "ぽ": "ポ", } _characters_all = {**_a_column, **_i_column, **_u_column, **_e_column, **_o_column} _u2i = { "う": "い", "く": "き", "す": "し", "つ": "ち", "ぬ": "に", "ふ": "ひ", "む": "み", "る": "り", } def _line_lower_result_replace( line, result, line_match: str, result_match: str, result_replacement: str ) -> str: if line_match in line.lower() and result_match in result: return result.replace(result_match, result_replacement) else: return result def _line_lower_result_replace_multi( line, result, line_match_lower: str, results_match: list, result_replacement: str ) -> str: for r in results_match: result = _line_lower_result_replace( line, result, line_match_lower, r, result_replacement ) additional_result_match = [] additional_lower_result_match = line_match_lower.split(" ") # All lower case: additional_result_match += [" ".join(additional_lower_result_match)] # All title case: additional_result_match += [ " ".join(e.title() for e in additional_lower_result_match) ] if " " in line_match_lower: # Dash connected case: additional_line_match_lower = line_match_lower.replace(" ", "-") result = _line_lower_result_replace_multi( line, result, additional_line_match_lower, results_match, result_replacement ) # First word title case: additional_result_match += [ additional_lower_result_match[0].title() + " " + " ".join(additional_lower_result_match[1:]) ] for x in additional_result_match: result = _line_lower_result_replace( line, result, line_match_lower, x, result_replacement ) return result def _line_result_replace( line, result, line_match: str, result_match: str, result_replacement: str ) -> str: if line_match in line and result_match in result: return result.replace(result_match, result_replacement) else: return result def _result_replace(result, result_match: str, result_replacement: str) -> str: if result_match in result: return result.replace(result_match, result_replacement) else: return result def _translate_cache_en2ja(line: str) -> str: """Return the cached Markdown translated result for English Markdown to Japanese Markdown. Args: line (str): The English source Markdown line. Returns: str: The cached Markdown translation result. """ if ( line.strip("\n").strip() == "No speaker notes." or line.strip("\n").strip() == "[no notes]" ): return "スピーカーのノートはありません。" if ( line.strip("\n").strip() == "## Overview" or line.strip("\n").strip() == "## Introduction" ): return "## 概要" if ( line.strip("\n").strip() == "### Overview" or line.strip("\n").strip() == "### Introduction" ): return "### 概要" if line.strip("\n").strip() == "## Notes": return "## ノート" if line.strip("\n").strip() == "### Notes": return "### ノート" if line.strip("\n").strip() == "## Duration": return "## 所要時間" if line.strip("\n").strip() == "### Duration": return "### 所要時間" if ( line.strip("\n").strip() == "## Solution Steps" or line.strip("\n").strip() == "## Steps" ): return "## ステップ" if ( line.strip("\n").strip() == "### Solution Steps" or line.strip("\n").strip() == "### Steps" ): return "### ステップ" if line.strip("\n").strip() == "## Outline": return "## アウトライン" if line.strip("\n").strip() == "## Domain Details": return "## 詳細" if line.strip("\n").strip() == "## Response Types": return "## 回答タイプ" if line.strip("\n").strip() == "# Objectives": return "# 目標" if line.strip("\n").strip() == "# Introduction to TiDB": return "# TiDBの概要" if line.strip("\n").strip() == "# Introduction to TiKV": return "# TiKVの概要" if line.strip("\n").strip() == "# Introduction to Placement Driver": return "# Placement Driverの概要" if line.strip("\n").strip() == "# Introduction to TiFlash": return "# TiFlashの概要" if line.strip("\n").strip() == "# Introduction to Raft": return "# Raftの概要" return None def pre_en2ja_auto_translation_md_prefix_aware(line_in: str) -> str: """ This routine currently is NOT in-use. Args: line_in (str): The English line to be translated. Returns: str: The English line refined for translating into Japanese. """ processed_line = None processed_line = line_in return processed_line def post_en2ja_auto_translation( line: str, auto_translated_line: str, source_line: str ) -> str: """ This routine will be called in a context WITHOUT the Markdown prefix such as "#, +, -, *", etc. :param line: The English source line. :param auto_translated_line: The translated line by auto translation service. :returns: The post processed result. """ result = auto_translated_line.strip("\n") print(util.speak_result("Begin Post Handling for Input: " + result)) # 1. Pre-processing before ですます processing. Refine needed, always needed. result = result.replace("この文を日本語に翻訳すると 「", "") result = re.sub("^「", "", result) result = re.sub("」$", "", result) result = re.sub("」。$", "", result) # してください」となります if result.strip("\n").endswith("してください」となります。"): result = result.replace("してください」となります。", "します。") if result.strip("\n").endswith("してください」となります；"): result = result.replace("してください」となります；", "します；") if result.strip("\n").endswith("してください」となります"): result = result.replace("してください」となります", "します") # 始めましょう if result.strip("\n").endswith("始めましょう。"): result = result.replace("始めましょう。", "始めます。") if result.strip("\n").endswith("始めましょう；"): result = result.replace("始めましょう；", "始めます；") if result.strip("\n").endswith("始めましょう"): result = result.replace("始めましょう", "始めます") # れてください if result.strip("\n").endswith("れてください。"): result = result.replace("れてください。", "れます。") if result.strip("\n").endswith("れてください；"): result = result.replace("れてください；", "れます；") if result.strip("\n").endswith("れてください"): result = result.replace("れてください", "れます") # してください, してください。」 if result.strip("\n").endswith("してください。"): result = result.replace("してください。", "します。") if result.strip("\n").endswith("してください。」"): result = result.replace("してください。」", "します。」") if result.strip("\n").endswith("してください；"): result = result.replace("してください；", "します；") if result.strip("\n").endswith("してください"): result = result.replace("してください", "します") # みてください if result.strip("\n").endswith("みてください。"): result = result.replace("みてください。", "みます。") if result.strip("\n").endswith("みてください；"): result = result.replace("みてください；", "みます；") if result.strip("\n").endswith("みてください"): result = result.replace("みてください", "みます") # しましょう if result.strip("\n").endswith("しましょう。"): result = result.replace("しましょう。", "します。") if result.strip("\n").endswith("しましょう；"): result = result.replace("しましょう；", "します；") if result.strip("\n").endswith("しましょう"): result = result.replace("しましょう", "します") # できない if result.strip("\n").endswith("できない。"): result = result.replace("できない。", "できません。") if result.strip("\n").endswith("できない；"): result = result.replace("できない；", "できません；") # ない if result.strip("\n").endswith("ない。"): result = result.replace("ない。", "ありません。") if result.strip("\n").endswith("ない；"): result = result.replace("ない；", "ありません；") # んだ if result.strip("\n").endswith("んだ。"): result = result.replace("んだ。", "みました。") if result.strip("\n").endswith("んだ；"): result = result.replace("んだ；", "みました；") # しかった - something passed if result.strip("\n").endswith("しかった。"): result = result.replace("しかった。", "しかったです。") if result.strip("\n").endswith("しかった；"): result = result.replace("しかった；", "しかったです；") print(util.speak_result("Post Phase 1 Completed: " + result)) # 2. The general sentence ending special handlings for "ですます体". result_raw = result.strip("\n").strip("。").rstrip("**").strip("。").rstrip("**") transformed_result_raw = [] for rr in result_raw.split("。"): if rr == None or len(rr) == 0: continue print(util.speak_result("2. ですます体 handling - rr-value: " + rr)) # か变动词(三类) if rr.endswith("来る"): rr = rr[0:-1] + "ます" # さ变动词(三类) elif rr.endswith("する"): rr = rr[0:-2] + "します" # 五段动词(一类): # A. [-1] is "る" and [-2] is in "a" column or "u" column or "o" column # B: [-1] is in "u" column and [-1] is not "る" elif rr[-1] == "る" and rr[-2] in ([*_a_column] + [*_u_column] + [*_o_column]): rr = rr[0:-1] + _u2i[rr[-1]] + "ます" elif ( rr[-1] == "る" and rr[-2] not in [*_characters_all] and util.is_char_cjk(rr[-2]) and editor.translate_jakz2hirakana(rr[-2]).strip("\n")[-1] in ([*_a_column] + [*_u_column] + [*_o_column]) ): rr = rr[0:-1] + _u2i[rr[-1]] + "ます" elif ( rr[-1] in [*_u_column] and rr[-1] != "る" and not rr.endswith("ます") and not rr.endswith("です") and not rr.endswith("しましょう") and not rr.endswith("でしょう") ): try: rr = rr[0:-1] + _u2i[rr[-1]] + "ます" except KeyError as ke: print(util.speak_result("One Exception Skipped: " + ke)) None # 一段动词(二类): # A. [-1] is "る" and [-2] is in "i" column or "e" column elif rr[-1] == "る" and rr[-2] in ([*_i_column] + [*_e_column]): rr = rr[0:-1] + "ます" elif ( rr[-1] == ("る") and rr[-2] not in [*_characters_all] and util.is_char_cjk(rr[-2]) and editor.translate_jakz2hirakana(rr[-2]).strip("\n")[-1] in ([*_i_column] + [*_e_column]) ): rr = rr[0:-1] + "ます" transformed_result_raw += [rr] result_raw = "。".join(transformed_result_raw) # End of 2. The general sentence ending special handlings for "ですます体". if result.endswith("。**\n"): result = result_raw + "。**\n" elif result.endswith("**。\n"): result = result_raw + "**。\n" elif result.endswith("。**"): result = result_raw + "。**" elif result.endswith("**。"): result = result_raw + "**。" elif result.endswith("。"): result = result_raw + "。" elif result.endswith("**"): result = result_raw + "**" elif result.endswith("\n"): result = result_raw + "\n" else: result = result_raw print(util.speak_result("Post Phase 2 Completed: " + result)) # 4. Correct case sensitive translation result by a static dictionary. # Load `line_match_cs, replacement, result_match_cs` from CSV. for e in _linematch_replace_resultmatch_cs[1:]: if not type(e[2]) == str: for i in e[2]: result = _line_result_replace(line, result, e[0], i, e[1]) else: result = _line_result_replace(line, result, e[0], e[2], e[1]) print(util.speak_result("Post Phase 4 Completed: " + result)) # 5. Correct case insensitive translation result. # Load `line_match_ci, replacement, result_match_cs` from CSV. for e in _linematch_replace_resultmatch_ci[1:]: result = _line_lower_result_replace_multi( line, result, e[0], e[2].split(","), e[1] ) print(util.speak_result("Post Phase 5 Completed: " + result)) # 6. Correct complex translation result. Goal keeper solutions for those issues that can not be solved in step 5 completely. if "use" in line.lower() and result.strip("\n").endswith("使え"): result = result.replace("使え", "使う") if "coprocessor" in line.lower() and "コプロセッサ" in result and "コプロセッサー" not in result: result = result.replace("コプロセッサ", "コプロセッサー") if "server" in line.lower() and "サーバ" in result and "サーバー" not in result: result = result.replace("サーバ", "サーバー") if "layer" in line.lower() and "レイヤ" in result and "レイヤー" not in result: result = result.replace("レイヤ", "レイヤー") if ( "candidates" in line.lower() and "候補" in result and "候補者" not in result ): # Following up fixing for "candidates". result = result.replace("候補", "Candidates") if ( "candidate" in line.lower() and "候補" in result and "候補者" not in result ): # Following up fixing for "candidate". result = result.replace("候補", "Candidate") if ( "role" in line.lower() and "役" in result and "役割" not in result ): # Following up fixing for "role(s)". result = result.replace("役", "ロール") if "lock" in line.lower() and (not "key" in line.lower()): result = _line_lower_result_replace_multi(line, result, "lock", ["鍵"], "ロック") if "row" in line.lower() and (not "raw" in line.lower()): if ("load" in line.lower() or "road" in line.lower()) and "ロード" in result: result = result.replace("ロード", "X_LROAD_PH_X") if "global" in line.lower() and "グローバル" in result: result = result.replace("グローバル", "X_GLOBAL_PH_X") if "workflow" in line.lower() and "ワークフロー" in result: result = result.replace("ワークフロー", "X_WORKFLOW_PH_X") result = ( result.replace("ロー", "行") .replace("X_LROAD_PH_X", "ロード") .replace("X_GLOBAL_PH_X", "グローバル") .replace("X_WORKFLOW_PH_X", "グローバル") ) if "tidb/tikv/pd" in line.lower(): result = re.sub("tidb/tikv/pd", "TiDB/TiKV/PD", result, flags=re.IGNORECASE) print(util.speak_result("Post Phase 6 Completed: " + result)) print(util.speak_result("Post Phase 6.1 Completed: " + line + " => " + result)) # 7. REMOVERS: # Remove "それは、" for "It is " prefixed bullet. if result.strip(" ").startswith("+") and line.strip(" ").strip("+ ").startswith( "It is " ): result = result.replace("それは、", "") # Remove the unwanted generated ** if "**" in result and "**" not in line: result = result.replace("**", "") # Remove the unwanted generated "「" and "」" if ( not line.startswith('"') and result.startswith("「") and not line.endswith('"') and result.endswith("」") ): result = result.strip("「") result = result.strip("」") # Remove the unwanted "(...)" if not "(" in source_line and not ")" in source_line: result = re.sub("\(.*\)", "", result) # Remove the unwanted generated "を日本語に翻訳します" result = result.replace("を日本語に翻訳します", "") result = result.replace("：「私は日本語を勉強しています。", "") # Remove the unwanted "です" at the end if re.match("^`[^`]+` *です[。]*$", result): result = result.replace("です", "") # Remove the unwanted "の" after `PARMS::` result = result.replace("`PARMS::` の ", "`PARMS::` ") print(util.speak_result("Post Phase 7 Completed: " + result)) return result def post_en2ja_auto_translation_md_prefix_aware(in_line: str, source_line: str) -> str: """ This routine will be called in a context with the Markdown prefix such as "#, +, -, *", etc. """ result = in_line if result.strip().startswith("+"): result = result.strip("。").strip("？") if result.strip().startswith("#"): result = result.strip("。").strip("？") if result.strip().startswith("["): result = result.strip("。").strip("？") if not "(" in source_line and not ")" in source_line: result = re.sub("\(.*\)", "", result) if not '"' in source_line: result = result.replace('"', "") return result def translate_en2ja_md_line( line: str, en_1gram_list: list, mode="standard", md4ppt=True, md_file_path=None, is_srt=False, ) -> str: """ Translate markdown line from EN to JA. Here goes the logic: 0. Translation cache. 1. Short-cut the line in case translation is not needed. 2. Pack the prefixed whitepsaces from the line, for example: `_____## Hello world` 3. Pack the prefixed `#`, `+`, `-`, `+`, `*`, `$numbering` (whitespace included) from the line. 4. Replace the inline codeline - `CODE` with placeholders. Translate the processed line. 5. Put the inline codeline back. 6. Post translation handling. 7-13. Unpack `#`, `+`, `-`, `+`, `*`, `$numbering` and prefixed whitspaces back to the line. """ # Source print(util.speak_result("SOURCE: [ " + line + " ]")) # Pre-process line = pre_en2ja_auto_translation_md_prefix_aware(line) print(util.speak_result("AFTER PRE-PROCESS MD-AWARE: [ " + line + " ]")) print(util.speak_result("BEFORE TRANSLATION: [ " + line + " ]")) # 0. Translation cache. cache_result = _translate_cache_en2ja(line) if cache_result != None: print(util.speak_result("Translate from cache: " + cache_result)) return cache_result # 1. Do nothing: Shortcut. if translator.is_skippable(line): for e in _errata_cs[1:]: line = _result_replace(line, e[0], e[1]) return line # Assign `line` to `compress_space_raw` to keep the line intact. compress_space_raw = line # 2. Pack the prefixed whitespace prefix_whitespace_pack = translator.pack_prefix_whitespace(compress_space_raw) compress_space_raw = prefix_whitespace_pack["processed_line"] # 3.1 Pack the prefixed hash prefix_hash_pack = translator.pack_prefix_hash(compress_space_raw) compress_space_raw = prefix_hash_pack["processed_line"] # 3.2 Pack the prefixed plus prefix_plus_pack = translator.pack_prefix_plus(compress_space_raw) compress_space_raw = prefix_plus_pack["processed_line"] # 3.3 Pack the prefixed minus prefix_minus_pack = translator.pack_prefix_minus(compress_space_raw) compress_space_raw = prefix_minus_pack["processed_line"] # 3.4 Pack the prefixed star prefix_star_pack = translator.pack_prefix_star(compress_space_raw) compress_space_raw = prefix_star_pack["processed_line"] # 3.5 Pack the prefixed number prefix_number_pack = translator.pack_prefix_number(compress_space_raw) compress_space_raw = prefix_number_pack["processed_line"] payload = compress_space_raw # 3.6 Do nothing: Shortcut. if not translator.is_skippable(compress_space_raw): # 4. Pack inline codeline inline_codeline_pack = translator.pack_inline_codeline(compress_space_raw) compress_space_raw = inline_codeline_pack["processed_line"] # Ready for translation before_translation = compress_space_raw.strip() # Translate if ( prefix_hash_pack["count"] > 0 ): # If the line is title line, convert it to lower case. before_translation = before_translation.lower() if not is_srt: translated_line = editor.translate_en2ja_by_openai(before_translation) else: translated_line = editor.translate_en2ja_by_openai_for_srt( before_translation ) print( util.speak_result( "BEFORE OPENAI TRANSLATION: [ " + before_translation + " ]" ) ) print( util.speak_result( "AFTER OPENAI TRANSLATION: [ " + translated_line.strip() + " ]" ) ) compress_space_raw = translated_line.strip() # 5. Inline code handling compress_space_raw = translator.unpack_inline_codeline( compress_space_raw, inline_codeline_pack ) # Cut-off the whitespaces if any compress_space_raw = compress_space_raw.strip() # 6. Post translate print( util.speak_result("BEFORE POST-TRANSLATION: [ " + compress_space_raw + " ]") ) compress_space_raw = post_en2ja_auto_translation( before_translation, compress_space_raw, payload ) print( util.speak_result("AFTER POST-TRANSLATION: [ " + compress_space_raw + " ]") ) # 7. Adding Hash title prefix back compress_space_raw = translator.unpack_prefix_hash( compress_space_raw, prefix_hash_pack, en_1gram_list, md4ppt ) # 8. Adding plus prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_plus_pack ) # 9. Adding minus prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_minus_pack ) # 10. Adding star prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_star_pack ) # 11. Adding number prefix back compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_number_pack ) # 12. Give back the prefix whitespace compress_space_raw = translator.unpack_prefix_common( compress_space_raw, prefix_whitespace_pack ) # 13. Correct the general results in all cases. for e in _errata_cs[1:]: compress_space_raw = _result_replace(compress_space_raw, e[0], e[1]) # Finally print(util.speak_result("AFTER TRANSLATION: [ " + compress_space_raw + " ]")) if md4ppt: compress_space_raw = post_en2ja_auto_translation_md_prefix_aware( compress_space_raw, line ) print( util.speak_result( "AFTER TRANSLATION MD-AWARE: [ " + compress_space_raw + " ]" ) ) return compress_space_raw