import re from datasdk.utils import utils import sqlite3 from o11ydiagnosis.subtasks.metricsconverter.log import log metrics_list = [ { "name": "latency_99", "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le))" }, { "name": "latency_999", "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le))" }, { "name": "tidb_server_connections", "expr": "sum(tidb_server_connections)" }, { "name": "tikv_store_size_bytes", "expr": 'sum(tikv_store_size_bytes{type="used"})' }, { "name": "tiflash_system_current_metric_StoreSizeUsed", "expr": 'sum(tiflash_system_current_metric_StoreSizeUsed)' }, { "name": 'latency', "expr": 'sum(increase(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (le)' }, { "name": "database_time_by_sql_phase_parse", "expr": 'sum(increase(tidb_session_parse_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_compile", "expr": 'sum(increase(tidb_session_compile_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_execute", "expr": 'sum(increase(tidb_session_execute_duration_seconds_sum{sql_type="general"}[2m]))' }, { "name": "database_time_by_sql_phase_get_token", "expr": 'sum(increase(tidb_server_get_token_duration_seconds_sum{}[2m]))/1000000' }, { "name": "database_time", "expr": 'sum(increase(tidb_server_handle_query_duration_seconds_sum{sql_type!="internal"}[2m]))' }, { "name": "tidb_cpu", 'expr': 'sum(rate(process_cpu_seconds_total{component="tidb"}[2m]))' }, { "name": "tidb_cpu_increase", 'expr': 'sum(increase(process_cpu_seconds_total{component="tidb"}[2m]))' }, { "name": "tikv_cpu", "expr": 'sum(rate(process_cpu_seconds_total{component="tikv"}[2m]))' }, { "name": "tikv_cpu_increase", "expr": 'sum(increase(process_cpu_seconds_total{component="tikv"}[2m]))' }, { "name": "tiflash_cpu", "expr": 'sum(rate(tiflash_proxy_process_cpu_seconds_total{component="tiflash"}[2m]))' }, { "name": "tiflash_cpu_increase", "expr": 'sum(increase(tiflash_proxy_process_cpu_seconds_total{component="tiflash"}[2m]))' }, { "name": "tidb_memory", "expr": 'sum(rate(process_resident_memory_bytes{component="tidb"}[2m]))' }, { "name": "tidb_memory_increase", "expr": 'sum(increase(process_resident_memory_bytes{component="tidb"}[2m]))' }, { "name": "tikv_memory", "expr": 'sum(rate(process_resident_memory_bytes{component="tikv"}[2m]))' }, { "name": "tikv_memory_increase", "expr": 'sum(increase(process_resident_memory_bytes{component="tikv"}[2m]))' }, { "name": "tiflash_memory", "expr": 'sum(rate(process_resident_memory_bytes{component="tiflash"}[2m]))' }, { "name": "tiflash_memory_increase", "expr": 'sum(increase(process_resident_memory_bytes{component="tiflash"}[2m]))' } ] create_table_sql_template = '''CREATE TABLE {}( metric_name CHAR(50) , metric_labels CHAR(200), metric_timestamp INT, metric_value INT );''' insert_sql_template = '''INSERT INTO {}('metric_name', 'metric_labels', 'metric_timestamp', 'metric_value') VALUES ( '{}', '{}', {}, '{}' );''' class Rule: def __init__(self, name, expr): self.name = name self.expr = expr def getMetrics(o11y_api, vendor, region, o11y_cluster_id, rules, start_ts, end_ts, step): dsPath = utils.getO11yDSDomain(vendor, region) + "/internal/metrics/{}".format(o11y_cluster_id) metricsData = dict() log.info("get metrics data from %d to %d", start_ts, end_ts-1) for rule in rules: partMetrics = o11y_api.getO11yMetrics( dsPath, rule.expr, start_ts, end_ts, step) if not partMetrics: continue for metric in partMetrics: lables = [] for k, v in metric["metric"].items(): if k == '__name__': continue lables.append('"{}":"{}"'.format(k, v)) lables = '{{{}}}'.format(','.join(lables)) name = '{}{}'.format(rule.name, lables) if metricsData.get(name) is None: metricsData[name] = [None] * \ ((end_ts - start_ts) // step + 1) for value in metric['values']: index = (value[0] - start_ts) // step metricsData[name][index] = value[1] return metricsData def write_metrics(metrics_data, cache_file_path, start_ts, step): conn = sqlite3.connect(cache_file_path) for metrics_header in metrics_data: pattern = r'{(.*?)}' match = re.search(pattern, metrics_header) if match: content = match.group() labels = content.replace('""', '') table = metrics_header.replace(labels, "") data = metrics_data[metrics_header] for index in range(0, len(data)): c = conn.cursor() insert_sql = insert_sql_template.format(table, table, labels, start_ts + step * index, data[index]) c.execute(insert_sql) else: # todo handle error continue conn.commit() conn.close()