import os, re, datetime, shutil, numpy as np from xmlrpc.client import Boolean from pingcap.pe.flash.markdown_processor import parse_markdown_folder from pingcap.pe.flash.util import load_1gram_terms_en, pretty_format_title # pattern chapters chain suggestion_and_views = {} pattern_suggestion_files = {} def _get_table_header(lang_code: str) -> list: title_suffix = None column_header = [] if lang_code[-2:] == "zh": title_suffix = "潜在的非标准写作" column_header = ["| 风险 | 建议 | 文件:实例 |\n"] elif lang_code[-2:] == "en": title_suffix = "Potential Non-Standard Narratives" column_header = ["| Risk | Suggestion | Filename:Instance |\n"] output_lines = [ "# " + pretty_format_title( os.path.basename(os.getcwd()).replace("-", " "), load_1gram_terms_en("pe") ) + " " + title_suffix + "\n" ] output_lines += ["+ " + str(datetime.datetime.now()) + "\n\n"] output_lines += column_header output_lines += ["| :-- | :-- | :-- |\n"] return output_lines def _seek_pattern_in_line( pattern: tuple, line: str, filename: str, course_folder_base: str ) -> Boolean: m = re.findall(pattern[0].lower(), line.strip().lower()) if m != None and len(m) > 0: # print('Seeking level 1 hit:',pattern[0].lower(),':',line.strip().lower()) m2 = re.findall(pattern[1].lower(), line.strip().lower()) if m2 != None and len(m2) > 0: None else: # print('Seeking level 2 hit:',pattern[0].lower(),':',line.strip().lower()) if pattern[2] == "Y": pattern_suggestion_files[ pattern[0].replace("\\", "") + ":=" + pattern[1].replace("\\", "") ] += [course_folder_base + "/" + filename] suggestion_and_views[ pattern[0].replace("\\", "") + ":=" + pattern[1].replace("\\", "") ] += [filename + ":" + line.strip()] return True else: return False def dump_suggestion_table( lang_code: str, negative_usage_pattern_list: list, auto_fix=False ) -> None: """ Dump a Markdown table for knowledge touched from a PE course structure. :param lang_code: en, en2zh, zh, zh2en. :param negative_usage_pattern_list: A list of examples of negative usage patterns. :param need to auto fix the negative pattern to suggested value? :returns: None. """ # Table header course_folder_base = "student-guide/view/" + lang_code course_chapter_folders = sorted(os.listdir(course_folder_base)) output_lines = _get_table_header(lang_code) # Driving pattern for pattern in negative_usage_pattern_list: if pattern[2] == "Y": pattern_suggestion_files.update( {pattern[0].replace("\\", "") + ":=" + pattern[1].replace("\\", ""): []} ) suggestion_and_views.update( {pattern[0].replace("\\", "") + ":=" + pattern[1].replace("\\", ""): []} ) # Driven chapters for chapter_folder in course_chapter_folders: chapter_folder_pattern_matches = re.findall( "^chapter-[0-9][0-9]", chapter_folder ) if ( chapter_folder_pattern_matches != None and len(chapter_folder_pattern_matches) > 0 and (not chapter_folder.endswith("-lms")) ): parsed_chapter = parse_markdown_folder( course_folder_base + "/" + chapter_folder ) for view in parsed_chapter: # Slide lines for slide_lines in view[0]: if ( slide_lines[1] != None and len(slide_lines[1].strip()) > 0 and not "/" in slide_lines[1] ): found_in_chapter = _seek_pattern_in_line( pattern, slide_lines[1], chapter_folder + "/" + view[2], course_folder_base, ) if found_in_chapter: print( pattern[0].replace("\\", ""), "\t-> Found in slide:", chapter_folder + "/" + view[2], ) # Speaker notes if nothing found in slide lines found_in_chapter = False for note_line in view[1]: if note_line != None and len(note_line.strip()) > 0: found_in_chapter = _seek_pattern_in_line( pattern, note_line, chapter_folder + "/" + view[2], course_folder_base, ) if found_in_chapter: print( pattern[0].replace("\\", ""), "\t-> Found in speaker notes:", chapter_folder + "/" + view[2], ) # Write result to file s_now = str(datetime.datetime.now()) dest_file = ( "metadata/raw/suggestion-" + os.path.basename(os.getcwd()) + "-" + lang_code + "-" + s_now + ".md" ) f = open(dest_file, "w") for k in suggestion_and_views.keys(): if suggestion_and_views[k] != None and len(suggestion_and_views[k]) > 0: patterns = k.split(":=") output_lines += [ "| " + patterns[0] + " | " + patterns[1] + " | " + ",".join(sorted(suggestion_and_views[k])) + " |\n" ] f.write("".join(output_lines)) f.close() if auto_fix: files = {} backup_path = "./stage/auto_fix_backup" for k in pattern_suggestion_files.keys(): if ( pattern_suggestion_files[k] != None and len(pattern_suggestion_files[k]) > 0 ): p = k.split(":=")[0] s = k.split(":=")[1] print(p, s, pattern_suggestion_files[k]) for fpath in pattern_suggestion_files[k]: if fpath in files.keys(): None else: try: print( "Backup", fpath, "to", re.sub("/[^/]*$", "", backup_path + "/" + fpath), ) os.makedirs( re.sub("/[^/]*$", "", backup_path + "/" + fpath) ) except FileExistsError: None shutil.copyfile(fpath, backup_path + "/" + fpath) f = open(fpath, "r") lines = f.read() f.close() files.update({fpath: lines}) for k in pattern_suggestion_files.keys(): if ( pattern_suggestion_files[k] != None and len(pattern_suggestion_files[k]) > 0 ): p = k.split(":=")[0] p_title = p.title() p_upper = p.upper() s = k.split(":=")[1] for fpath in pattern_suggestion_files[k]: files[fpath] = ( files[fpath] .replace(p, s) .replace(p_title, s) .replace(p_upper, s) ) for fpath in files: print("Fixing", fpath) f = open(fpath, "w") f.write(files[fpath]) f.close() print("\nPlease check", dest_file, "\n")