""" * Version: 0.1.0.0.0 * Purposes: * Dump a Markdown which contains a keywords table for PE course. * System parameters: * sys_parameter 1: The language code, for example: en, zh, en2zh, zh2en. """ import sys, re, os from pingcap.pe.flash.course_keyword_seeker import dump_keyword_table from pingcap.pe.flash.markdown_processor import parse_markdown_folder # Not used in dynamic keywords marking style k_map = { "k_en_list": [ "_tidb_rowid", "apply-max-batch-size", "apply-pool-size", "ANSI_QUOTES", "AUTO_INCREMENT", "AUTO_RANDOM", "B-tree", "balance-leader-scheduler", "balance-region-scheduler", "block-cache", "Alarm system", "Non-stop upgrades", "pessimistic locking", "expression index", "expression push down", "invisible index", "ClusterCache", "committer-concurrency", "commit_st", "Connector/J", "Connector/Python", "coprocessor_process", "coproesssor_wait", "CURRENT_TIMESTAMP", "big transaction", "window function", "re-election", "Storage thresholds", "DATA_LOCK_WAITS", "DistSQL", "dirty_ratio", "dirty_background_ratio", "dr-replicas", "single point of failure", "Isodepth histogram", "Regular maintenance", "hotspots", "Read process", "slow query", "multiple replicas", "enforce_mpp", "evict-leader", "ERROR_FOR_DIVISION_BY_ZERO", "force-priority", "Non-clustered index", "Non-clustered table", "Distributed transaction", "Distributed clock", "Distributed system", "unevenly distributed", "partitioned table", "Partition pruning", "load balancing", "Geo-partition", "grpc-connection-count", "isolation level", "high-space-ratio", "hot-region-scheduler", "hot-region-schedule-limit", "mixed workload", "JDBC", "unplanned downtime", "cluster upgrade", "clustered index", "fast recovery", "LAST_INSERT_ID()", "leader-schedule-limit", "leader_weight", "location-labels", "LOCALTIME", "LOCALTIMESTAMP", "low-space-ratio", "LSM tree", "optimistic locking", "columnar storage", "temporary table", "2 phase commit", "composite index", "logical optimization", "max-merge-region-size", "max-merge-region-keys", "max-proc", "MemTable", "merge-schedule-limit", "oltp_read_write", "oltp_read_only", "noatime", "nodiratime", "ONLY_FULL_GROUP_BY", "Online DDL", "oom-action", "oom-use-tmp-storage", "PartialAggregation", "patrol-region-interval", "PIPES_AS_CONCAT", "PITR", "PointGet", "PRE_SPLIT_REGIONS", "primary-replicas", "Prometheus", "readpool.unified.max-thread-count", "region-schedule-limit", "region_weight", "RegionHeartbeat", "raft-max-inflight-msgs", "RootTask", "secondary index", "scheduler-worker-pool-size", "sql_mode", "SHARD", "SPM", "SplitRegion", "split.qps-threshold", "split.byte-threshold", "SST", "storage.block-cache.capacity", "store-max-batch-size", "store-pool-size", "StoreHeartbeat", "StreamingResult", "SHARD_ROW_ID_BITS", "flashback table", "transaction control", "dead lock", "temporary table", "tmp-storage-path", "tmp-storage-quota", "TiFlash", "TIDB_HOT_REGIONS", "TIDB_TRX", "TIDB_PK_TYPE", "tidb_ddl_error_count_limit", "tidb_ddl_reorg_priority", "tidb_auto_analyze_start_time", "tidb_auto_analyze_end_time", "tidb_index_join_batch_size", "tidb_store_limit", "tidb_distsql_scan_concurrency", "tidb_index_lookup_concurrency", "tidb_build_stats_concurrency", "tidb_hash_join_concurrency", "tidb_index_lookup_join_concurrency", "tidb_ddl_reorg_worker_cnt", "tidb_init_chunk_size", "tidb_max_chunk_size", "tidb-mem-quota-query", "tidb_index_serial_scan_concurrency", "tidb_ddl_reorg_batch_size", "tidb_snapshot", "tidb_scatter_region", "tidb_expensive_query_time_threshold", "token-limit", "tolerant-size-ratio", "tsFuture", "UnifyRead", "Userspace", "prepared statement", "autocommit", "write_buffer_size", "predicate pushdown", "physical optimization", "unique key", "unique index", "system table", "engine smart selection", ], "k_zh_list": [ "_tidb_rowid", "2PC", "apply-max-batch-size", "apply-pool-size", "ANSI_QUOTES", "AUTO_INCREMENT", "AUTO_RANDOM", "B-tree", "Backoff", "Batch", "balance-leader-scheduler", "balance-region-scheduler", "block-cache", "报警系统", "备份", "补丁升级", "不停机升级", "编译", "悲观锁", "并发度", "表达式索引", "表达式下推", "表的统计信息", "不可见索引", "CAST", "CDC", "ClusterCache", "CNF", "Collation", "Coordinator", "committer-concurrency", "commit_st", "Concurrency", "Connector/J", "Connector/Python", "Coprocessor", "coprocessor_process", "coproesssor_wait", "CopTask", "cpufreq", "CURRENT_TIMESTAMP", "拆分大事务", "窗口函数", "重新选举", "创建角色", "创建索引的速度", "存储空间阈值", "存储引擎", "DATA_LOCK_WAITS", "DBaaS", "CFQ", "DISTINCT", "DistSQL", "dirty_ratio", "dirty_background_ratio", "dmctl", "dmesg", "DNF", "dr-replicas", "大页", "大量DML", "单点故障", "调度流程", "等深直方图", "定点数", "定期维护", "定义恢复点目标", "定义恢复时间目标", "定位热点", "读取流程", "读取慢排查", "读写分离", "多副本的强一致性和高可用性", "enforce_mpp", "evict-leader", "ERROR_FOR_DIVISION_BY_ZERO", "Failover", "force-priority", "服务器混布", "非聚簇表的主键", "非聚簇索引", "非聚簇表", "分页", "分布式事务", "分布式时钟", "分布式系统", "分布不均衡", "分区表", "分区剪裁", "浮点数", "覆盖索引", "负载均衡", "赋权", "Geo-partition", "Grafana", "grpc-connection-count", "gRPC", "隔离级别", "high-space-ratio", "HLC", "hot-region-scheduler", "hot-region-schedule-limit", "HTAP", "恢复时效性", "混合负载场景", "JDBC", "计划内系统不可用", "计划外系统不可用", "集群升级", "集群拓扑", "计算加速", "聚合消除", "绝对时间", "聚簇索引", "聚簇表", "开源社区", "快速恢复", "扩展性", "LAST_INSERT_ID()", "leader-schedule-limit", "leader_weight", "Lightning", "location-labels", "LOCALTIME", "LOCALTIMESTAMP", "low-space-ratio", "LSM", "垃圾回收", "乐观锁", "列存储", "列剪裁", "临时磁盘", "两阶段提交", "连接顺序调整", "复合索引", "流量控制", "流量可视化", "流式计算场景", "逻辑时钟", "逻辑优化", "集合运算", "max-merge-region-size", "max-merge-region-keys", "max-proc", "MemTable", "merge-schedule-limit", "MVCC", "慢查询", "NewSQL", "oltp_read_write", "oltp_read_only", "noatime", "nodiratime", "NoSQL", "NONCLUSTERED", "NOOP", "ONLY_FULL_GROUP_BY", "OOM", "oom-action", "oom-use-tmp-storage", "PartialAggregation", "patrol-region-interval", "pd-ctl", "Playground", "PIPES_AS_CONCAT", "PITR", "PointGet", "PRE_SPLIT_REGIONS", "primary-replicas", "Prometheus", "Python", "频繁更新", "清理集群", "权限", "全局发号器", "readpool.unified.max-thread-count", "region-schedule-limit", "region_weight", "RegionHeartbeat", "Raft", "raft-max-inflight-msgs", "RootTask", "二级索引", "热点", "热点打散", "人为错误", "日期格式", "软件故障", "scheduler-worker-pool-size", "sql_mode", "SHARD", "SPM", "SplitRegion", "split.qps-threshold", "split.byte-threshold", "SST", "storage.block-cache.capacity", "store-max-batch-size", "store-pool-size", "StoreHeartbeat", "StreamingResult", "SHARD_ROW_ID_BITS", "闪回", "设置账号密码", "升级常见问题", "时间戳", "时区的修改", "实时同步", "事务控制", "事务的去中心化", "数据对象", "数据格式", "数据持久化", "数据恢复前置条件", "数据回放平台", "数据库恢复", "数据库写入慢", "数据写入流程", "数据一致性", "数据组织模型", "审计与分析", "索引分析", "索引选择", "死锁", "tmp-storage-path", "tmp-storage-quota", "TiFlash", "TIDB_HOT_REGIONS", "TIDB_TRX", "TIDB_PK_TYPE", "tidb_ddl_error_count_limit", "tidb_ddl_reorg_priority", "tidb_auto_analyze_start_time", "tidb_auto_analyze_end_time", "tidb_index_join_batch_size", "tidb_store_limit", "tidb_distsql_scan_concurrency", "tidb_index_lookup_concurrency", "tidb_build_stats_concurrency", "tidb_hash_join_concurrency", "tidb_index_lookup_join_concurrency", "tidb_ddl_reorg_worker_cnt", "tidb_init_chunk_size", "tidb_max_chunk_size", "tidb-mem-quota-query", "tidb_index_serial_scan_concurrency", "tidb_ddl_reorg_batch_size", "tidb_snapshot", "tidb_scatter_region", "tidb_expensive_query_time_threshold", "token-limit", "tolerant-size-ratio", "tsFuture", "统计信息的基本组成", "UnifyRead", "Userspace", "写入瓶颈", "相关子查询", "相对时间", "硬件故障", "预编译语句", "执行计划", "自动提交", "write_buffer_size", "谓词下推", "外连接消除", "网络延迟", "物理时钟", "物理优化", "无状态", "系统表", "系统参数的作用域", "系统读写热点查询", "系统慢日志查询", "向量时钟", "线程池", "写偏斜", "验证集群", "业务隔离", "异步复制", "异步切同步", "元数据存储", "优化器算法", "有效索引", "在线 DDL", "在线扩容", "在线缩容", "智能选择", "直方图", "自动分片", "最左匹配原则", "阻塞查询", ], } if __name__ == "__main__": lang_code = sys.argv[1] course_folder_base = "student-guide/view/" + lang_code course_chapter_folders = sorted(os.listdir(course_folder_base)) knowledge_point_keywords = [] knowledge_point_pattern = "__\*.*\*__" for chapter_folder in course_chapter_folders: chapter_folder_pattern_matches = re.findall( "^chapter-[0-9][0-9]", chapter_folder ) if ( chapter_folder_pattern_matches != None and len(chapter_folder_pattern_matches) > 0 and (not chapter_folder.endswith("-lms")) ): parsed_chapter = parse_markdown_folder( course_folder_base + "/" + chapter_folder ) for view in parsed_chapter: for slide_line in view[0]: if ( slide_line[1] != None and len(slide_line[1].strip()) > 0 and not "/" in slide_line[1] ): matches = re.findall(knowledge_point_pattern, slide_line[1]) if matches != None and len(matches) > 0: for m in matches: knowledge_point_keywords += [ m.replace("__*", "").replace("*__", "") ] dump_keyword_table(lang_code, sorted(set(knowledge_point_keywords)))