import os import sys import unittest from diagnosis.preprocess.slowlog.preprocessor import * log = '''db-tidb-0,1680587792,"# Time: 2023-04-04T05:56:31.456041802Z # Txn_start_ts: 440556005641158673 # Conn_ID: 2608613327147347345 # Query_time: 1.357866546 # Parse_time: 0.000162409 # Compile_time: 0.001546155 # Rewrite_time: 0.000661093 # Optimize_time: 0.00069593 # Wait_TS: 0.000008708 # DB: adwords # Is_internal: false # Digest: digest1 # Num_cop_tasks: 15 # Mem_max: 120895 # Prepared: false # Plan_from_cache: false # Plan_from_binding: false # Has_more_results: false # KV_total: 8.675286099000001 # PD_total: 0.000005319 # Backoff_total: 0 # Write_sql_response_total: 0 # Result_rows: 0 # Succ: true # IsExplicitTxn: false # Plan_digest: 1232434234 select * from test;" db-tidb-0,1680587792,"# Time: 2023-04-04T05:56:31.390116921Z # Txn_start_ts: 440556005667110926 # Conn_ID: 2608613327147358551 # Query_time: 1.199196262 # Parse_time: 0.000074138 # Compile_time: 0.000727736 # Rewrite_time: 0.000284946 # Optimize_time: 0.000351605 # Wait_TS: 0.000416949 # DB: adwords # Is_internal: false # Digest: digest2 # Stats: AW_CampaignPerformanceReport:440535343345958932 # Num_cop_tasks: 12 # Mem_max: 128031 # Prepared: false # Plan_from_cache: false # Plan_from_binding: false # Has_more_results: false # KV_total: 4.632504744 # PD_total: 0.0004104 # Backoff_total: 0 # Write_sql_response_total: 0 # Result_rows: 0 # Succ: true # IsExplicitTxn: false show tables;" ''' class TestSplit(unittest.TestCase): def test_execute(self): cfg = SplitConfig("split", "split", [], [], '"\n', False) p = Split(cfg, [log]) result = [] try: result = p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(0, len(result)) else: print(result) self.assertNotEqual(2, len(result)) class TestParse(unittest.TestCase): def test_execute(self): data = ['db-tidb-0,1680587792,' '"# Time: 2023-04-04T05:56:31.456041802Z\n' '# Txn_start_ts: 440556005641158673\n# Conn_ID: 2608613327147347345\n' '# Query_time: 1.357866546\n# Parse_time: 0.000162409\n' '# Optimize_time: 0.00069593\n# DB: adwords\n' '# Digest: digest1\n# Mem_max: 120895\n' 'select * from test;', 'db-tidb-0,1680587792,"' '# Time: 2023-04-04T05:56:31.390116921Z\n' '# Txn_start_ts: 440556005667110926\n' '# Query_time: 1.199196262\n# Parse_time: 0.000074138\n' '# DB: adwords\n# Digest: digest2\n# Mem_max: 128031\n' 'show tables;'] cfg = ParseConfig("parse", "paser", [], [], r"#\s*(\w+)\s*:\s*(.*?)\n") p = Parse(cfg, [data]) result = [] try: result = p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(0, len(result)) else: print(result) self.assertEqual(2, len(result)) self.assertEqual(9, len(result[0])) class TestConvertToDict(unittest.TestCase): def test_execute(self): data = [[('Time', '2023-04-04T05:56:31.456041802Z'), ('Txn_start_ts', '440556005641158673'), ('Conn_ID', '2608613327147347345'), ('Query_time', '1.357866546'), ('Parse_time', '0.000162409'), ('Optimize_time', '0.00069593'), ('DB', 'adwords'), ('Digest', 'digest1'), ('Mem_max', '120895')], [('Time', '2023-04-04T05:56:31.390116921Z'), ('Txn_start_ts', '440556005667110926'), ('Query_time', '1.199196262'), ('Parse_time', '0.000074138'), ('DB', 'adwords'), ('Digest', 'digest2'), ('Mem_max', '128031')]] cfg = ConvertToDictConfig("convert", "convert", [], []) p = ConvertToDict(cfg, [data]) result = [] try: result = p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(0, len(result)) else: print(result) self.assertEqual(2, len(result)) self.assertEqual(9, len(result[0])) class TestMapping(unittest.TestCase): def test_execute(self): data = [{'Time': '2023-04-04T05:56:31.456041802Z', 'Txn_start_ts': '440556005641158673', 'Conn_ID': '2608613327147347345', 'Query_time': '1.357866546', 'Parse_time': '0.000162409', 'Optimize_time': '0.00069593', 'DB': 'adwords', 'Digest': 'digest1', 'Mem_max': '120895'}, {'Time': '2023-04-04T05:56:31.390116921Z', 'Txn_start_ts': '440556005667110926', 'Query_time': '1.199196262', 'Parse_time': '0.000074138', 'DB': 'adwords', 'Digest': 'digest2', 'Mem_max': '128031'}] cfg = MappingConfig("mapping", "mapping", [], [], ["time", "db", "digest", "query_time", "mem_max"], ["Time", "DB", "Digest", "Query_time", "Mem_max"]) p = Mapping(cfg, [data]) result = [] try: result = p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(0, len(result)) else: print(result) self.assertEqual(2, len(result)) self.assertEqual(5, len(result[0])) class TestMergeMap(unittest.TestCase): def test_execute(self): data1 = [{'time': '2023-04-04T05:56:31.456041802Z', 'db': 'adwords', 'digest': 'digest1', 'query_time': '1.357866546', 'mem_max': '120895'}, {'time': '2023-04-04T05:56:31.390116921Z', 'db': 'adwords', 'digest': 'digest2', 'query_time': '1.199196262', 'mem_max': '128031'}] data2 = [{'tidb_instance': 'tidb-db-0'}, {'tidb_instance': 'tidb-db-0'}] cfg = MergeConfig("merge", "merge", [], []) p = Merge(cfg, [data1, data2]) result = [] try: result = p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(0, len(result)) else: print(result) self.assertEqual(2, len(result)) self.assertEqual(6, len(result[0])) class TestCreateSqlite(unittest.TestCase): def test_execute(self): data = [{'time': '2023-04-04T05:56:31.456041802Z', 'db': 'adwords', 'digest': 'digest1', 'query_time': '1.357866546', 'mem_max': '120895', 'tidb_instance': 'tidb-db-0'}, {'time': '2023-04-04T05:56:31.390116921Z', 'db': 'adwords', 'digest': 'digest2', 'query_time': '1.199196262', 'mem_max': '128031', 'tidb_instance': 'tidb-db-0'}] cfg = CreateSqliteConfig("sqlite", "sqlite", [], [], "./test.db", "slowlog") p = CreateSqlite(cfg, [data]) try: p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(False, os.path.exists("./test.db")) else: self.assertEqual(True, os.path.exists("./test.db")) finally: os.remove("./test.db") class TestCreateCountSqlite(unittest.TestCase): def test_execute(self): data = [{"time": "2023-03-07T07:31:42.828651591Z", "tidb_instance": "tidb:db-0", "digest": "digest1"}, {"time": "2023-03-07T07:32:42.828651591Z", "tidb_instance": "tidb:db-0", "digest": "digest1"}, {"time": "2023-03-07T07:31:42.828651591Z", "tidb_instance": "tidb:db-1", "digest": "digest1"}, {"time": "2023-03-07T07:31:42.828651591Z", "tidb_instance": "tidb:db-0", "digest": "digest2"}, {"time": "2023-03-07T07:32:42.828651591Z", "tidb_instance": "tidb:db-0", "digest": "digest2"}] test_db_path = "./test_count.db" cfg = CreateCountSqliteConfig("create", "create", [], [], 1678145500, 1678145600, 20, test_db_path, "slowlog_count") p = CreateCountSqlite(cfg, [data]) try: p.execute() except Exception: print("get err", sys.exc_info()[1]) self.assertEqual(False, os.path.exists(test_db_path)) else: self.assertEqual(True, os.path.exists(test_db_path)) finally: if os.path.exists(test_db_path): os.remove(test_db_path) if __name__ == '__main__': unittest.main()