', "`", 1)
line = input_line.replace('
', " `").replace("
", "` ")
line = line.replace(" *__", "*__").replace(" **", "**")
return line
def _post_process_line_after_translation_v2(input_line: str) -> str:
return input_line
def dump_markdown(view: tuple, output_file_path: str) -> None:
"""
Writes structured tuple as a markdown file.
:param view: Input tuple parsed by parse_markdown().
:param output_file_path: The path for the output Markdown file.
:returns: None
"""
if platform.system() == "Windows":
f = open(output_file_path, "x", encoding="utf-8")
else:
f = open(output_file_path, "x")
markdown_content = ""
# 1. Slide
slide_lines = view[0]
for slide_line in slide_lines:
if slide_line[0] in ["chapter_title:main", "title:slide"]:
markdown_content = markdown_content + "# " + slide_line[1] + "\n"
elif slide_line[0] in ["chapter_subtitle:main"]:
markdown_content = markdown_content + "## " + slide_line[1] + "\n"
elif slide_line[0] == "bulletin:0":
markdown_content = (
markdown_content + "+ " + _pre_dump_bulletin_line(slide_line[1])
)
elif slide_line[0] == "bulletin:1":
markdown_content = (
markdown_content + " + " + _pre_dump_bulletin_line(slide_line[1])
)
elif slide_line[0] == "bulletin:2":
markdown_content = (
markdown_content + " + " + _pre_dump_bulletin_line(slide_line[1])
)
elif slide_line[0] == "bulletin:3":
markdown_content = (
markdown_content + " + " + _pre_dump_bulletin_line(slide_line[1])
)
elif slide_line[0] == "bulletin:4":
markdown_content = (
markdown_content + " + " + _pre_dump_bulletin_line(slide_line[1])
)
elif slide_line[0].startswith("picture") or slide_line[0].startswith(
"background"
):
markdown_content = markdown_content + "\n" + slide_line[2] + "\n"
elif slide_line[0].startswith("link:url"):
markdown_content = markdown_content + "\n" + slide_line[2] + "\n"
elif slide_line[0] == "textbox:common":
markdown_content = markdown_content + slide_line[1]
elif slide_line[0] == "code_area:begin":
markdown_content = markdown_content + "```" + slide_line[1]
elif slide_line[0] in ["code_area:end"]:
markdown_content = markdown_content + "```" + "\n"
elif slide_line[0] == "white_space:common":
markdown_content = markdown_content
elif slide_line[0] == "code_area:body":
markdown_content = markdown_content + slide_line[1]
elif slide_line[0] == "knowledge_point:keyword":
markdown_content = markdown_content + slide_line[1]
# 2. Table
if len(view) > 3 and view[3] != None:
table_lines = view[3]
for table_line in table_lines:
if table_line[0] == "table:header":
markdown_content += "|"
for col in table_line[1]:
markdown_content += " " + col.strip() + " |"
markdown_content += "\n"
# markdown_content += "|"
# for col in table_line[1]:
# markdown_content += " :-- |"
# markdown_content += "\n"
elif table_line[0] == "table:column_settings":
markdown_content += "|"
for col in table_line[1]:
markdown_content += " " + col.strip() + " |"
markdown_content += "\n"
elif table_line[0] == "table:row":
markdown_content += "|"
for col in table_line[1]:
markdown_content += " " + col.strip() + " |"
markdown_content += "\n"
# 3. Notes
note_lines = view[1]
if note_lines != None:
markdown_content = markdown_content + "\n" + "-" * 60 + "\n"
for note_line in note_lines:
markdown_content = markdown_content + note_line # + '\n' testing
f.write(markdown_content)
f.close()
def dump_markdown_folder(chapter: list, output_folder_path: str) -> None:
"""
Write Markdown files to a folder as a chapter.
A Markdown folder maps to a chapter (a list of tuples), which in turn maps to a PowerPoint file.
:param chapter: Input list of tuples parsed by parse_markdown_folder().
:param output_file_path: The path for the output Markdown file.
:returns: None
"""
os.makedirs(output_folder_path)
for view in chapter:
print("markdown_processor: output_folder_path", output_folder_path)
dump_markdown(view, output_folder_path + "/" + view[2])
def write_markdown(content: str, output_file_path: str, mode="standard") -> None:
if platform.system() == "Windows":
f = open(output_file_path, "x", encoding="utf-8")
else:
f = open(output_file_path, "x")
f.write(content)
f.close()
def write_markdown_folder(
chapter: list, output_folder_path: str, mode="standard"
) -> None:
os.makedirs(output_folder_path)
for view in chapter:
if mode == "standard":
print(">>> markdown_processor: output_folder_path", output_folder_path)
write_markdown(view[0], output_folder_path + "/" + view[1], mode)
def translate_chapter_v2(
chapter_path: str,
options: str,
terms_name: str,
en_1gram_list: list,
profile_name=None,
skip_count=0,
mode="standard",
engine="openai",
) -> list:
if platform.system() == "Windows":
source_markdown_file_names = sorted(
[chapter_path + str.strip(" \ ") + f for f in os.listdir(chapter_path)]
)
else:
source_markdown_file_names = sorted(
[chapter_path + "/" + f for f in os.listdir(chapter_path)]
)
chapter = []
view_count = 0
for f in source_markdown_file_names:
if f.endswith(".md") or f.endswith(".srt"):
view_count += 1
if view_count > skip_count:
chapter += [
translate_view_v2(
f,
options,
terms_name,
en_1gram_list,
profile_name,
mode,
engine,
)
]
return chapter # The chapter - collection of views of markdown.
def translate_view_v2(
markdown_file_path: str,
options: str,
terms_name: str,
en_1gram_list: list,
profile_name=None,
mode="standard",
engine="openai",
) -> tuple:
"""
Single view (slide) translation with disk I/O. This routine works with other v2 functions.
Notes:
1. 1st element in view: a list of (slide_line_type, slide_line_content).
2. 2nd element in view: a list of string for the speaker notes.
3. 3rd element in view: Markdown file basename.
:param view: The view need to be translated.
:param options: The translation options, such as: zh2en, en2zh, ... etc.
:param terms_name: Specifiy the terminology dictionary name, format: terms-
--. Such as:such as terms-pe-zh-en, terms-tse-zh-en, terms-doc-zh-en, ... etc.
:param profile_name: AWS or CSP profile name prvides the basic translation service. None means the default profile.
:returns: Translated view. View structure description:
(translated_content,markdown_basename)
"""
file_basename = os.path.basename(markdown_file_path)
if platform.system() == "Windows":
f = open(markdown_file_path, "r", encoding="UTF-8")
else:
f = open(markdown_file_path, "r")
lines = f.read()
f.close()
result = []
translation_mute_zone = (
False # some parts does not need to be translated, such as code blocks.
)
for source_line in lines.split("\n"):
line_result = "\n"
if (
re.match("^ *``", source_line)
and len(re.findall("``", source_line)) % 2 == 1
and translation_mute_zone
):
translation_mute_zone = False
line_result = source_line
elif (
re.match("^ *``", source_line)
and len(re.findall("``", source_line)) % 2 == 1
and not translation_mute_zone
):
translation_mute_zone = True
line_result = source_line
elif not re.match("^ *``", source_line) and translation_mute_zone:
line_result = source_line
elif (not translation_mute_zone) and (
re.match("^[0-9]*$", source_line.strip())
or re.match("^[ -0123456789:,>]*$", source_line.strip())
):
line_result = source_line
elif not translation_mute_zone:
if engine == "openai":
# Pause for 2 seconds to give openai a break.
print(util.speak_result("Source line: " + source_line))
time.sleep(2)
if options.endswith("ja"):
line_result = util_en2ja.translate_en2ja_md_line(
source_line, en_1gram_list, mode, True, markdown_file_path
)
elif options.endswith("zh"):
line_result = util_en2zh.translate_en2zh_md_line(
source_line, en_1gram_list, mode, True, markdown_file_path
)
elif options.endswith("en"):
line_result = util_zh2en.translate_zh2en_md_line(
source_line, en_1gram_list, mode, True, markdown_file_path
)
result += [line_result]
translation_mute_zone = False
return ("\n".join(result), file_basename)
def refine_view(
markdown_file_path: str,
en_1gram_list: list,
) -> tuple:
"""
Single view (slide) refine with disk I/O.
Notes:
1. 1st element in view: a list of (slide_line_type, slide_line_content).
2. 2nd element in view: a list of string for the speaker notes.
3. 3rd element in view: Markdown file basename.
:param terms_name: Specifiy the terminology dictionary name, format: terms---. Such as:such as terms-pe-zh-en, terms-tse-zh-en, terms-doc-zh-en, ... etc.
:param profile_name: AWS or CSP profile name prvides the basic translation service. None means the default profile.
:returns: Translated view. View structure description:
(translated_content,markdown_basename)
"""
file_basename = os.path.basename(markdown_file_path)
if platform.system() == "Windows":
f = open(markdown_file_path, "r", encoding="UTF-8")
else:
f = open(markdown_file_path, "r")
lines = f.read()
f.close()
result = []
translation_mute_zone = (
False # some parts does not need to be refined, such as code blocks.
)
for source_line in lines.split("\n"):
line_result = "\n"
if (
re.match("^ *``", source_line)
and len(re.findall("``", source_line)) % 2 == 1
and translation_mute_zone
):
translation_mute_zone = False
line_result = source_line
elif (
re.match("^ *``", source_line)
and len(re.findall("``", source_line)) % 2 == 1
and not translation_mute_zone
):
translation_mute_zone = True
line_result = source_line
elif not re.match("^ *``", source_line) and translation_mute_zone:
line_result = source_line
elif (not translation_mute_zone) and (
re.match("^[0-9]*$", source_line.strip())
or re.match("^[ -0123456789:,>]*$", source_line.strip())
):
line_result = source_line
elif not translation_mute_zone:
line_result = refine_english_line_in_md(
source_line, "standard", True, markdown_file_path
)
result += [line_result]
translation_mute_zone = False
return ("\n".join(result), file_basename)
def parse_markdown(file_path: str) -> tuple:
"""
1. File basename `01title.md` is treated to presentation chapter title slide.
2. File basename `99close.md` declaratively means the end of chapter.
:param file_path: Path of the Markdown file.
:returns: The parsed view. Parsed view structure description:
(
list:[
(slide_line_type, slide_line_content),...
],
list:[str:note_line,...],
str:markdown_basename,
list:[str:table_line,...],
)
"""
file_basename = os.path.basename(file_path)
table_lines_list = []
parsed_lines_list = []
notes_lines_list = []
if platform.system() == "Windows":
f = open(file_path, "r", encoding="UTF-8")
else:
f = open(file_path, "r")
lines = f.read()
f.close()
# Chapter title
if file_path.endswith("01title.md"):
for line in lines.splitlines(False):
if re.match("^# ", line):
parsed_lines_list += [("chapter_title:main", line.replace("# ", ""))]
elif re.match("^## ", line):
parsed_lines_list += [
("chapter_subtitle:main", line.replace("## ", "").strip())
]
focus = False
for note in lines.splitlines(True):
if re.match("^\-+$", note):
focus = True
elif focus:
notes_lines_list += [note]
else:
continue
return (parsed_lines_list, notes_lines_list, file_basename)
# Normal slides
out_of_code_area = True
notes_area = False
for l in lines.splitlines(True):
line = l.rstrip(" ")
if len(line) > 0:
if (
re.match("^```[0-9]*$", line) and out_of_code_area and not notes_area
): # Code area begin
out_of_code_area = False
parsed_lines_list += [
("code_area:begin", line.replace("```", ""))
] # Save the font size or None
elif (
re.match("^```$", line) and not out_of_code_area and not notes_area
): # Code area end
out_of_code_area = True
parsed_lines_list += [("code_area:end", None)]
elif (
re.match("^\-+$", line) and out_of_code_area and not notes_area
): # Speaker notes begin, stop the slide parsing
notes_area = True
break
elif (
re.match("^# ", line) and out_of_code_area and not notes_area
): # Slide title
parsed_lines_list += [("title:slide", line.replace("# ", ""))]
elif (
re.match("^\+ ", line) and out_of_code_area and not notes_area
): # Level 0 B
parsed_lines_list += [("bulletin:0", line.replace("+ ", ""))]
elif (
re.match("^ \+ ", line) and out_of_code_area and not notes_area
): # Level 1 B
parsed_lines_list += [("bulletin:1", line.replace(" + ", ""))]
elif (
re.match("^ \+ ", line) and out_of_code_area and not notes_area
): # Level 2 B
parsed_lines_list += [("bulletin:2", line.replace(" + ", ""))]
elif (
re.match("^ \+ ", line) and out_of_code_area and not notes_area
): # Level 3 B
parsed_lines_list += [("bulletin:3", line.replace(" + ", ""))]
elif (
re.match("^ \+ ", line) and out_of_code_area and not notes_area
): # Level 4 B
parsed_lines_list += [("bulletin:4", line.replace(" + ", ""))]
elif (
re.match("^!\[`smart`", line) and out_of_code_area and not notes_area
): # Picture smart positioning
parsed_lines_list += [
(
"picture:smart",
line.split("(")[1].replace(")", "").strip(),
line.strip(),
)
]
elif (
re.match("^!\[`center`", line) and out_of_code_area and not notes_area
): # Picture center
parsed_lines_list += [
(
"picture:center",
line.split("(")[1].replace(")", "").strip(),
line.strip(),
)
]
elif (
re.match("^!\[`left`", line) and out_of_code_area and not notes_area
): # Picture left
parsed_lines_list += [
(
"picture:left",
line.split("(")[1].replace(")", "").strip(),
line.strip(),
)
]
elif (
re.match("^!\[`right`", line) and out_of_code_area and not notes_area
): # Picture right
parsed_lines_list += [
(
"picture:right",
line.split("(")[1].replace(")", "").strip(),
line.strip(),
)
]
elif (
re.match("^!\[`background`", line)
and out_of_code_area
and not notes_area
): # Picture full as background
parsed_lines_list += [
(
"background:full",
line.split("(")[1].replace(")", "").strip(),
line.strip(),
)
]
elif (
re.match("^\[", line) and out_of_code_area and not notes_area
): # Hyper link
c_link_name = re.findall("^\[(.*)\]", line)
parsed_lines_list += [("link:url", c_link_name[0], line.strip())]
elif (
re.match("^\|", line) and out_of_code_area and not notes_area
): # Table line (only one table is supported)
if len(table_lines_list) == 0:
table_lines_list += [("table:header", line.split("|")[1:-1])]
elif (":--" in line) or ("--:" in line):
table_lines_list += [
("table:column_settings", line.split("|")[1:-1])
]
elif (not ":--" in line) and (not "--:" in line):
table_lines_list += [("table:row", line.split("|")[1:-1])]
elif (
re.match("^ +$", line) and out_of_code_area and not notes_area
): # White space
parsed_lines_list += [("white_space:common", " ")]
elif not out_of_code_area and not notes_area:
parsed_lines_list += [("code_area:body", line)]
elif (
re.match("__\*.*\*__", line) and out_of_code_area and not notes_area
): # Knowledge point
parsed_lines_list += [("knowledge_point:keyword", line)]
elif not notes_area: # Logically else
None
if notes_area: # Do we have speaker notes?
if platform.system() == "Windows":
f = open(file_path, "r", encoding="UTF-8")
else:
f = open(file_path, "r")
focus_notes = f.read()
f.close()
focus = False
for note in focus_notes.splitlines(True):
if re.match("^\-+$", note):
focus = True
elif focus:
notes_lines_list += [note]
else:
continue
return (
parsed_lines_list,
notes_lines_list,
file_path,
table_lines_list,
) # The view - (slide lines, speaker notes, file name, table rows)
def parse_markdown_folder(folder_path: str) -> list:
"""
:param folder_path: The path to the Markdown file folders.
:returns: Parsed chapter as a list of tuples.
Chapter structure description:
[([(slide_line_type, slide_line_content),...],[note_line,...],markdown_basename)]
"""
if platform.system() == "Windows":
source_markdown_file_names = sorted(
[folder_path + str.strip(" \ ") + f for f in os.listdir(folder_path)]
)
else:
source_markdown_file_names = sorted(
[folder_path + "/" + f for f in os.listdir(folder_path)]
)
chapter = []
for f in source_markdown_file_names:
if f.endswith(".md") and not "^" in f:
chapter += [parse_markdown(f)]
return chapter # The chapter - collection of slides
def refine_english_line_in_md(
line: str, en_1gram_list: list, mode="standard", md4ppt=True, md_file_path=None
) -> str:
"""
Refine markdown line in English.
Here goes the logic:
1. Short-cut the line in case refine is not needed.
2. Pack the prefixed whitepsaces from the line, for example: `_____## Hello world`
3. Pack the prefixed `#`, `+`, `-`, `+`, `*`, `$numbering` (whitespace included) from the line.
4. Replace the inline codeline - `CODE` with placeholders.
Translate the processed line.
5. Put the inline codeline back.
7-12. Unpack `#`, `+`, `-`, `+`, `*`, `$numbering` and prefixed whitspaces back to the line.
"""
# Source
print(util.speak_result("SOURCE: [ " + line + " ]"))
# 1. Do nothing: Shortcut.
if translator.is_skippable(line):
return line
# Assign `line` to `compress_space_raw` to keep the line intact.
compress_space_raw = line
# 2. Pack the prefixed whitespace
prefix_whitespace_pack = translator.pack_prefix_whitespace(compress_space_raw)
compress_space_raw = prefix_whitespace_pack["processed_line"]
# 3.1 Pack the prefixed hash
prefix_hash_pack = translator.pack_prefix_hash(compress_space_raw)
compress_space_raw = prefix_hash_pack["processed_line"]
# 3.2 Pack the prefixed plus
prefix_plus_pack = translator.pack_prefix_plus(compress_space_raw)
compress_space_raw = prefix_plus_pack["processed_line"]
# 3.3 Pack the prefixed minus
prefix_minus_pack = translator.pack_prefix_minus(compress_space_raw)
compress_space_raw = prefix_minus_pack["processed_line"]
# 3.4 Pack the prefixed star
prefix_star_pack = translator.pack_prefix_star(compress_space_raw)
compress_space_raw = prefix_star_pack["processed_line"]
# 3.5 Pack the prefixed number
prefix_number_pack = translator.pack_prefix_number(compress_space_raw)
compress_space_raw = prefix_number_pack["processed_line"]
# 4. Pack inline codeline
inline_codeline_pack = translator.pack_inline_codeline(compress_space_raw)
compress_space_raw = inline_codeline_pack["processed_line"]
# Ready for refine
before_refine = compress_space_raw.strip()
# Refine
refined_line = editor.refine_en_sentence_by_openai(before_refine)
print(util.speak_result("BEFORE OPENAI REFINE: [ " + before_refine + " ]"))
print(util.speak_result("AFTER OPENAI REFINE: [ " + refined_line + " ]"))
compress_space_raw = refined_line.strip()
# 5. Inline code handling
compress_space_raw = translator.unpack_inline_codeline(
compress_space_raw, inline_codeline_pack
)
# Cut-off the whitespaces if any
compress_space_raw = compress_space_raw.strip()
print(util.speak_result("BEFORE POST-REFINE: [ " + compress_space_raw + " ]"))
# 7. Adding Hash title prefix back
compress_space_raw = translator.unpack_prefix_hash(
compress_space_raw, prefix_hash_pack, en_1gram_list, md4ppt
)
# 8. Adding plus prefix back
compress_space_raw = translator.unpack_prefix_common(
compress_space_raw, prefix_plus_pack
)
# 9. Adding minus prefix back
compress_space_raw = translator.unpack_prefix_common(
compress_space_raw, prefix_minus_pack
)
# 10. Adding star prefix back
compress_space_raw = translator.unpack_prefix_common(
compress_space_raw, prefix_star_pack
)
# 11. Adding number prefix back
compress_space_raw = translator.unpack_prefix_common(
compress_space_raw, prefix_number_pack
)
# 12. Give back the prefix whitespace
compress_space_raw = translator.unpack_prefix_common(
compress_space_raw, prefix_whitespace_pack
)
print(util.speak_result("AFTER REFINE: [ " + compress_space_raw + " ]"))
return compress_space_raw