import dataclasses from diagnosis.preprocess.slowlog import preprocessor as slowlog from diagnosis.preprocess.metrics import preprocessor as metrics from diagnosis.preprocess.event import preprocessor as events @dataclasses.dataclass class TaskConfig: cluster_id: str auth0_id: str auth0_secret: str clinic_token: str start_ts: int end_ts: int path: str metrics_sqlite_name: str slowlog_sqlite_name: str events_sqlite_name: str skip_tiflash: bool skip_slowlog: bool task_info: dict metrics_rules = [ { "name": 'tidb_executor_statement_total', "expr": "sum(rate(tidb_executor_statement_total{}[2m])) by (type)" }, { "name": "qps", "expr": "sum(rate(tidb_server_query_total{}[2m])) by (type) " }, { "name": "tidb_server_execute_error_total", "expr": "sum(increase(tidb_server_execute_error_total{}[1m])) by (type)" }, { "name": "latency_90", "expr": "histogram_quantile(0.90, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le))" }, { "name": "latency_99", "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le))" }, { "name": "latency_999", "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le))" }, { "name": "tidb_server_connections", "expr": "sum(tidb_server_connections)" }, { "name": "tikv_store_size_bytes", "expr": 'sum(tikv_store_size_bytes{type="used"})' }, { "name": "tiflash_system_current_metric_StoreSizeUsed", "expr": 'sum(tiflash_system_current_metric_StoreSizeUsed)' }, { "name": 'latency', "expr": 'sum(increase(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le)' }, { "name": "database_time_by_sql_phase_parse", "expr": 'sum(increase(tidb_session_parse_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_compile", "expr": 'sum(increase(tidb_session_compile_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_execute", "expr": 'sum(increase(tidb_session_execute_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_get_token", "expr": 'sum(increase(tidb_server_get_token_duration_seconds_sum{}[2m]))/1000000' }, { "name": "database_time", "expr": 'sum(increase(tidb_server_handle_query_duration_seconds_sum{sql_type!="internal"}[2m]))' }, { "name": "tidb_cpu", 'expr': 'sum(rate(process_cpu_seconds_total{component="tidb"}[2m]))' }, { "name": "tidb_cpu_increase", 'expr': 'sum(increase(process_cpu_seconds_total{component="tidb"}[2m]))' }, { "name": "tikv_cpu", "expr": 'sum(rate(process_cpu_seconds_total{component="tikv"}[2m]))' }, { "name": "tikv_cpu_increase", "expr": 'sum(increase(process_cpu_seconds_total{component="tikv"}[2m]))' }, { "name": "tiflash_cpu", "expr": 'sum(rate(tiflash_proxy_process_cpu_seconds_total{component="tiflash"}[2m]))' }, { "name": "tiflash_cpu_increase", "expr": 'sum(increase(tiflash_proxy_process_cpu_seconds_total{component="tiflash"}[2m]))' }, { "name": "tidb_memory", "expr": 'sum(rate(process_resident_memory_bytes{component="tidb"}[2m]))' }, { "name": "tidb_memory_increase", "expr": 'sum(increase(process_resident_memory_bytes{component="tidb"}[2m]))' }, { "name": "tikv_memory", "expr": 'sum(rate(process_resident_memory_bytes{component="tikv"}[2m]))' }, { "name": "tikv_memory_increase", "expr": 'sum(increase(process_resident_memory_bytes{component="tikv"}[2m]))' }, { "name": "tiflash_memory", "expr": 'sum(rate(process_resident_memory_bytes{component="tiflash"}[2m]))' }, { "name": "tiflash_memory_increase", "expr": 'sum(increase(process_resident_memory_bytes{component="tiflash"}[2m]))' } ] metrics_rules_more = [ { "name": "tidb_executor_statement_total", "expr": "sum(rate(tidb_executor_statement_total{}[2m])) by (type)" }, { "name": "tidb_server_query_total", "expr": "sum(rate(tidb_server_query_total{type=~\"StmtExecute|StmtFetch|StmtClose|StmtPrepare" "|StmtSendLongData|Query\"}[2m])) by (type) " }, { "name": "tidb_server_execute_error_total", "expr": "sum(increase(tidb_server_execute_error_total{}[1m])) by (type)" }, { "name": "tidb_server_handle_query_duration_seconds_999", "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le))" }, { "name": "latency", "expr": 'sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le)' }, { "name": "latency_by_sql_type_999", "expr": 'histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le,sql_type))' }, { "name": "latency_by_sql_type_99", "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le,sql_type))" }, { "name": "tidb_server_connections", "expr": "sum(tidb_server_connections)" }, { "name": "tidb_server_connections_idle_duration_99", "expr": 'histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{}[2m])) by (le))' }, { "name": "tidb_session_transaction_duration_seconds_999", "expr": 'histogram_quantile(0.999, sum(rate(tidb_session_transaction_duration_seconds_bucket{}[2m])) by (le, txn_mode))' }, { "name": "tidb_session_transaction_ops", "expr": 'sum(rate(tidb_session_transaction_duration_seconds_count{}[2m])) by (type, txn_mode)' }, { "name": "database_time", "expr": 'sum(tidb_server_tokens{})' }, { "name": "database_time_by_sql_type", "expr": 'sum(rate(tidb_server_handle_query_duration_seconds_sum{sql_type!="internal"}[2m])) by (sql_type)' }, { "name": "database_time_by_sql_phase_parse", "expr": 'sum(rate(tidb_session_parse_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_compile", "expr": 'sum(rate(tidb_session_compile_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_execute", "expr": 'sum(rate(tidb_session_execute_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_get_token", "expr": 'sum(rate(tidb_server_get_token_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "tidb_cpu", 'expr': 'sum(rate(node_cpu_seconds_total{mode!="idle",component="tidb"}[2m]))' }, { "name": "tikv_cpu", "expr": 'sum(rate(node_cpu_seconds_total{mode!="idle",component="tikv"}[2m]))' }, { "name": "tiflash_cpu", "expr": 'sum(rate(node_cpu_seconds_total{mode!="idle",component="tiflash"}[2m]))' }, { "name": "tidb_memory", "expr": 'sum(node_memory_MemTotal_bytes{component="tidb"} - node_memory_MemAvailable_bytes{component="tidb"})' }, { "name": "tikv_memory", "expr": 'sum(node_memory_MemTotal_bytes{component="tikv"} - node_memory_MemAvailable_bytes{component="tikv"})' }, { "name": "tiflash_memory", "expr": 'sum(node_memory_MemTotal_bytes{component="tiflash"} - node_memory_MemAvailable_bytes{component="tiflash"})' }, { "name": "tikv_store_size_bytes", "expr": 'sum(tikv_store_size_bytes{type="used"})' }, { "name": "tiflash_system_current_metric_StoreSizeUsed", "expr": 'sum(tiflash_system_current_metric_StoreSizeUsed)' }, { "name": "IOPS", "expr": 'irate(node_disk_reads_completed_total{}[2m]) + irate(node_disk_writes_completed_total{}[2m])' } ] slowlog_table_fields = ['Time', 'DB', 'Digest', 'Query_time', 'Mem_max', 'User_Host', 'Conn_ID', 'Disk_max', 'Txn_start_ts', 'Write_sql_response_total', 'Backoff_total', 'KV_total', 'PD_total', 'Result_rows', 'Parse_time', 'Compile_time', 'Rewrite_time', 'Wait_TS', 'Is_internal', 'Index_names', 'Stats', 'Succ', 'Backoff_time', 'Prepared', 'Plan_from_cache', 'Plan_from_binding', 'Has_more_results', 'Write_keys', 'Write_size', 'Num_cop_tasks', 'Cop_proc_avg', 'Cop_wait_avg', 'Cop_backoff', 'Cop_time', 'Commit_time', 'Write_keys', 'Write_size', 'Prewrite_time', 'Prewrite_region', 'Get_commit_ts_time', 'Local_latch_wait_time', 'Preproc_subqueries', 'Preproc_subqueries_time', 'Plan', 'Plan_digest', 'Optimize_time', 'Get_snapshot_time', 'Rocksdb_block_read_time', 'IsSyncStatsFailed', 'IsExplicitTxn'] slowlog_fields = ['Time', 'DB', 'Digest', 'Query_time', 'Mem_max', 'User@Host', 'Conn_ID', 'Disk_max', 'Txn_start_ts', 'Write_sql_response_total', 'Backoff_total', 'KV_total', 'PD_total', 'Result_rows', 'Parse_time', 'Compile_time', 'Rewrite_time', 'Wait_TS', 'Is_internal', 'Index_names', 'Stats', 'Succ', 'Backoff_time', 'Prepared', 'Plan_from_cache', 'Plan_from_binding', 'Has_more_results', 'Write_keys', 'Write_size', 'Num_cop_tasks', 'Cop_proc_avg', 'Cop_wait_avg', 'Cop_backoff', 'Cop_time', 'Commit_time', 'Write_keys', 'Write_size', 'Prewrite_time', 'Prewrite_region', 'Get_commit_ts_time', 'Local_latch_wait_time', 'Preproc_subqueries', 'Preproc_subqueries_time', 'Plan', 'Plan_digest', 'Optimize_time', 'Get_snapshot_time', 'Rocksdb_block_read_time', 'IsSyncStatsFailed', 'IsExplicitTxn'] slowlog_split_type = "split" slowlog_parse_type = "parse" slowlog_convert_to_dict_type = "convert_to_dict" slowlog_mapping_type = "mapping" slowlog_merge_map_type = "merge_map" slowlog_parse_sql_type = "parse_sql" slowlog_analyze_plan_type = "analyze_plan" slowlog_trace_type = "trace" # slowlog_convert_to_tuple_type = "convert_to_tuple" slowlog_create_sqlite_type = "create_sqlite_slowlog" slowlog_create_count_sqlite_type = "create_count_sqlite_slowlog" metrics_read_file_type = "read_metrics_file" metrics_clean_data_type = "clean_data_metrics" metrics_convert_to_tuple_type = "convert_metrics_to_tuple" metrics_create_sqlite_type = "create_sqlite_metrics" events_read_file_type = "read_events_file" events_convert_to_tuple_type = "convert_events_to_tuple" events_create_sqlite_type = "create_sqlite_events" TypeMap: dict = { slowlog_split_type: slowlog.Split, slowlog_parse_type: slowlog.Parse, slowlog_convert_to_dict_type: slowlog.ConvertToDict, slowlog_mapping_type: slowlog.Mapping, slowlog_merge_map_type: slowlog.Merge, slowlog_parse_sql_type: slowlog.ParseSQL, slowlog_analyze_plan_type: slowlog.AnalyzePlan, slowlog_trace_type: slowlog.TraceSQL, # slowlog_convert_to_tuple_type: slowlog.ConvertToTuple, slowlog_create_sqlite_type: slowlog.CreateSqlite, slowlog_create_count_sqlite_type: slowlog.CreateCountSqlite, metrics_read_file_type: metrics.ReadFile, metrics_clean_data_type: metrics.CleanData, metrics_convert_to_tuple_type: metrics.ConvertMetricsToTuple, metrics_create_sqlite_type: metrics.CreateMetricsSqlite, events_read_file_type: events.ReadFile, events_convert_to_tuple_type: events.ConvertEventsToTuple, events_create_sqlite_type: events.CreateEventsSqlite }