import os, re, datetime from xmlrpc.client import Boolean from pingcap.pe.flash.markdown_processor import parse_markdown_folder from pingcap.pe.flash.util import load_1gram_terms_en, pretty_format_title # Keyword chapters chain keyword_and_chapters = {} def _get_table_header(lang_code: str) -> list: title_suffix = None column_header = [] if lang_code[-2:] == "zh": title_suffix = "关键字快照" column_header = ["| 关键字 | 章节:幻灯片号.[幻灯片标题] | 关键字 | 章节:幻灯片号.[幻灯片标题] |\n"] elif lang_code[-2:] == "en": title_suffix = "Keywords Snapshot" column_header = [ "| Keyword | Chapter:Slide_No.[Slide_Title] | Keyword | Chapter:Slide_No.[Slide_Title] |\n" ] output_lines = [ "# " + pretty_format_title( os.path.basename(os.getcwd()).replace("-", " "), load_1gram_terms_en("pe") ) + " " + title_suffix + "\n" ] s_now = str(datetime.datetime.now()) output_lines += ["+ " + s_now + "\n\n"] output_lines += column_header output_lines += ["| :-- | :-- | :-- | :-- |\n"] return output_lines def _seek_keyword_in_line( keyword: str, line: str, chapter_folder: str, view: tuple ) -> Boolean: if "\\" in keyword: keyword = keyword.replace("\\", "\\\\") m = re.findall("__\*" + keyword.strip().lower() + "\*__", line.strip().lower()) if m != None and len(m) > 0: slide_lines = view[0] for slide_line in slide_lines: if slide_line[0] in ["title:slide"]: slide_title = slide_line[1] internal_slide_no = view[2].replace(".md", "") if "\\\\" in keyword: keyword = keyword.replace("\\\\", "\\") keyword_and_chapters[keyword] += [ chapter_folder.strip().replace("chapter-0", "").replace("chapter-", "") + ":" + str(internal_slide_no) + ".[" + slide_title.strip() + "]" ] return True else: return False def dump_keyword_table(lang_code: str, keyword_list: list) -> None: """ Dump a Markdown table for knowledge touched from a PE course structure. :param lang_code: en, en2zh, zh, zh2en. :param keyword_list: A list of keywords as the knowledge point. :returns: None. """ # Table header course_folder_base = "student-guide/view/" + lang_code course_chapter_folders = sorted(os.listdir(course_folder_base)) output_lines = _get_table_header(lang_code) # Driving keyword for keyword in keyword_list: keyword_and_chapters.update({keyword: []}) # Driven chapters for chapter_folder in course_chapter_folders: found_in_chapter = False chapter_folder_pattern_matches = re.findall( "^chapter-[0-9][0-9]", chapter_folder ) if ( chapter_folder_pattern_matches != None and len(chapter_folder_pattern_matches) > 0 and (not chapter_folder.endswith("-lms")) ): parsed_chapter = parse_markdown_folder( course_folder_base + "/" + chapter_folder ) for view in parsed_chapter: # Found in slide lines, proceed to next chapter # if found_in_chapter: # break # Slide lines for slide_lines in view[0]: if ( slide_lines[1] != None and len(slide_lines[1].strip()) > 0 and not "/" in slide_lines[1] ): found_in_chapter = _seek_keyword_in_line( keyword, slide_lines[1], chapter_folder, view ) if found_in_chapter: print(keyword, "\t-> Found in slide of", chapter_folder) break if not found_in_chapter: # Speaker notes if nothing found in slide lines for note_line in view[1]: if note_line != None and len(note_line.strip()) > 0: found_in_chapter = _seek_keyword_in_line( keyword, note_line, chapter_folder, view ) if found_in_chapter: print( keyword, "\t-> Found in speaker notes of", chapter_folder, ) break # Write result to file s_now = str(datetime.datetime.now()) dest_file = ( "metadata/raw/keywords-" + os.path.basename(os.getcwd()) + "-" + lang_code + "-" + s_now + ".md" ) f = open(dest_file, "w") key_index = 0 for k in keyword_and_chapters.keys(): if keyword_and_chapters[k] != None and len(keyword_and_chapters[k]) > 0: key_index += 1 if key_index % 2 == 0: output_lines += [ " | " + k + " | " + "; ".join(sorted(keyword_and_chapters[k])) + " |\n" ] else: output_lines += [ "| " + k + " | " + "; ".join(sorted(keyword_and_chapters[k])) ] f.write("".join(output_lines)) f.close() print("\nPlease check", dest_file, "\n")