# Application configuration feishu: documentUrl: wetch: "https://pingcap.feishu.cn/wiki/ROQhwcsRiiefUUkK9EqcYocKndc" bingo: "https://pingcap.feishu.cn/wiki/RuKgwAodziRDgWkHqgkcbcZFnEK" ue: "https://pingcap.feishu.cn/wiki/WHCOweb27izodDkSzgYcuuFqn7b" documentID: wetech: dashboard: "DKR9dsjLXow5CVxXUPscrpEln7g" blockId: "GiO2d0IPzoDMU9xFdlUcfHPfnUd" detail: saas: "FPxJd494Pocz2gxPqb5cOADNn7c" order: "HNr3d6S6woBUNDxb9GSc4MKSnTg" wallet: "WPU3d07ApowbFoxsQ6fc8VJZnlh" goGame: "YHC4dpV9ZoyOJDxo4ldcj9bznZf" bingo: dashboard: "SYBwdlqBMoSdJZxkSSAc4XqEnIf" blockId: "" detail: main: "El4xdWNNeoaTqfxm5nOclZGGn6e" ue: dashboard: "RrLKdaRnDorK6uxWMbycm1jVntl" blockId: "" detail: main: "CNC2duiTMoDP5Sx01nGczo9Znfd" receiveID: wetech: - "oc_271b5e3f25d764f1fc7fa591d10dd963" - "oc_f2dae01fdee204a8b75f5ed5f4808fb5" bingo: - "oc_271b5e3f25d764f1fc7fa591d10dd963" - "oc_d544427c8d3fa5c7eb923ffa2050063c" ue: - "oc_271b5e3f25d764f1fc7fa591d10dd963" - "oc_1e9a771840c771816271a057cb088ae6" cardTemplate: templateID: "AAqzQWytHTMxj" templateVersionName: "1.1.1" clinic: # Base URL for TiDB Clinic baseUrl: "https://clinic.pingcap.com/portal/dashboard/cloud/ngm.html" webHost: "https://clinic.pingcap.com" clusterMetricsTemplateUrl: "https://www.ds.%s.aws.observability.tidbcloud.com/internal/metrics/%s" clusterMetricsTemplateUrlTiDBX: "https://www.ds.%s.aws.observability.tidbcloud.com/internal/v2/metrics/%s" compareQuery: - name: "latency_p999_all" primaryQuery: 'histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster"}[$interval])) by (le))*1000' - name: "latency_p999" primaryQuery: 'histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster",sql_type =~"Insert|Delete|Select|Update"}[$interval])) by (le,sql_type))*1000' - name: "latency_p99_all" primaryQuery: 'histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster"}[$interval])) by (le))*1000' - name: "latency_p99" primaryQuery: 'histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster",sql_type =~"Insert|Delete|Select|Update"}[$interval])) by (le,sql_type))*1000' - name: "latency_p95_all" primaryQuery: 'histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster"}[$interval])) by (le))*1000' - name: "latency_p95" primaryQuery: 'histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster",sql_type =~"Insert|Delete|Select|Update"}[$interval])) by (le,sql_type))*1000' - name: "latency_avg_all" primaryQuery: 'sum(rate(tidb_server_handle_query_duration_seconds_sum{tidb_cluster_id="$tidb_cluster"}[$interval])) / (sum(rate(tidb_server_handle_query_duration_seconds_count{tidb_cluster_id="$tidb_cluster"}[$interval])) > 0) * 1000' - name: "latency_p999_by_type" primaryQuery: 'histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster",sql_type =~"Insert|Delete|Select|Update"}[$interval])) by (le,sql_type))*1000' - name: "latency_avg_by_type" primaryQuery: 'sum(rate(tidb_server_handle_query_duration_seconds_sum{tidb_cluster_id="$tidb_cluster",sql_type=~"Insert|Select|Update|Delete"}[$interval])) by (sql_type) / (sum(rate(tidb_server_handle_query_duration_seconds_count{tidb_cluster_id="$tidb_cluster",sql_type=~"Insert|Select|Update|Delete"}[$interval])) by (sql_type) > 0) * 1000' - name: "cps" primaryQuery: 'sum(rate(tidb_server_query_total{tidb_cluster_id="$tidb_cluster",result="OK"}[$interval])) by (result)' - name: "qps_all" primaryQuery: 'sum(rate(tidb_executor_statement_total{tidb_cluster_id="$tidb_cluster"}[$interval]))' - name: "qps_by_type" interval: "1m" primaryQuery: 'sum(rate(tidb_executor_statement_total{tidb_cluster_id="$tidb_cluster",type=~"Insert|Delete|Select|Update"}[$interval])) by (type)' - name: "cpu_tidb" primaryQuery: 'avg ((sum by (tidb_cluster_id, instance,cluster) (rate(process_cpu_seconds_total{tidb_cluster_id="$tidb_cluster",component="tidb"}[$interval]))/(sum by (tidb_cluster_id, instance,cluster) (tidb_server_maxprocs{tidb_cluster_id="$tidb_cluster"}) > 0)))' - name: "cpu_tikv" primaryQuery: 'avg(sum by(instance)(rate(process_cpu_seconds_total{tidb_cluster_id="$tidb_cluster",component="tikv"}[$interval]))/(sum by(instance)(tikv_server_cpu_cores_quota{tidb_cluster_id="$tidb_cluster"}) > 0))' - name: "cpu_tiflash" primaryQuery: 'avg((sum by (instance) (rate(tiflash_proxy_process_cpu_seconds_total{tidb_cluster_id="$tidb_cluster",component=~".*tiflash"}[$interval]))/(sum by (instance) (tiflash_system_current_metric_LogicalCPUCores{tidb_cluster_id="$tidb_cluster"}) > 0)))' - name: "component_num" type: "o11y" primaryQuery: 'sum(label_replace(kube_pod_info{namespace="tidb$namespace", pod=~"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+"},"component", "$1$3", "pod", "db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+")) by (component)' - name: "tikv_performance" primaryQuery: 'histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster", type=~"kv_prewrite|coprocessor"}[$interval])) by (le, type))*1000' - name: "tiflash_performance" primaryQuery: 'histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{tidb_cluster_id="$tidb_cluster",type = "run_mpp_task"}[$interval])) by (le, type))*1000' - name: "affected_rows_by_type" primaryQuery: 'sum by (sql_type)(rate(tidb_server_affected_rows{tidb_cluster_id="$tidb_cluster"}[$interval])) OR sum by (sql_type)(rate(tidb_executor_affected_rows{tidb_cluster_id="$tidb_cluster"}[$interval]))' # dataplan metrics - name: "memory_tikv" primaryQuery: 'max(max by (pod) ((node_memory_MemTotal_bytes{job="node-exporter"}-node_memory_MemFree_bytes{job="node-exporter"}-node_memory_Buffers_bytes{job="node-exporter"}-node_memory_Cached_bytes{job="node-exporter"})/node_memory_MemTotal_bytes{job="node-exporter"}*on(instance) group_left(pod)label_replace(kube_pod_info{namespace=~"tidb$namespace", pod=~".*tikv-.*"},"instance", "$1", "node", "(.*)")) * 100)' - name: "memory_tidb" primaryQuery: 'max(max by (pod) (((node_memory_MemTotal_bytes{job="node-exporter"}-node_memory_MemFree_bytes{job="node-exporter"}-node_memory_Buffers_bytes{job="node-exporter"}-node_memory_Cached_bytes{job="node-exporter"})/node_memory_MemTotal_bytes{job="node-exporter"})*on(instance) group_left(pod)label_replace(kube_pod_info{namespace=~"tidb$namespace", pod=~".*tidb-.*"},"instance","$1","node", "(.*)")) * 100)' - name: "iops" primaryQuery: 'max by (namespace) (avg by (namespace,pod,volume_id) (rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$interval])*on(instance) group_left(pod,namespace)label_replace(kube_pod_info{namespace=~"tidb$namespace"},"instance", "$1", "node", "(.*)"))>0)' - name: "iops_exceed_max" primaryQuery: 'max by (namespace) (avg by (namespace,pod,volume_id) (rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$interval])*on(instance) group_left(pod,namespace)label_replace(kube_pod_info{namespace=~"tidb$namespace",pod=~".*tikv.*"},"instance", "$1", "node", "(.*)")))' - name: "iops_exceed_avg" primaryQuery: 'sum(avg by (namespace,pod,volume_id) (rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$interval]) * on(instance) group_left(pod) label_replace(kube_pod_info{namespace=~"tidb$namespace", pod=~".*tikv.*"}, "instance", "$1", "node", "(.*)")))/(count(kube_pod_info{namespace="tidb$namespace", pod=~".*tikv.*"}) > 0)' o11y: auth0TokenUrl: "https://tidb-soc2.us.auth0.com/oauth/token" auth0Audience : "https://tidb-soc2.us.auth0.com/api/v2/" o11yMetadataBase: "http://www.gs.us-west-2.aws.observability.tidbcloud.com/api/v1/clusters" cpuResource: - name: "TiDB_CPU_P90" primaryQuery: 'quantile(0.9,sum(rate(process_cpu_seconds_total{component="tidb"}[$rate])) by (instance)) by (cluster_id)' - name: "TiDB_CPU_P80" primaryQuery: 'quantile(0.8,sum(rate(process_cpu_seconds_total{component="tidb"}[$rate])) by (instance)) by (cluster_id)' - name: "TiDB_CPU" primaryQuery: 'sum(rate(process_cpu_seconds_total{component="tidb"}[$rate])) by (cluster_id)' - name: "TiKV_CPU_P90" primaryQuery: 'quantile(0.9, sum(rate(process_cpu_seconds_total{component=~".*tikv"}[$rate])) by (instance)) by(cluster_id)' - name: "TiKV_CPU_P80" primaryQuery: 'quantile(0.8, sum(rate(process_cpu_seconds_total{component=~".*tikv"}[$rate])) by (instance)) by(cluster_id)' - name: "TiKV_CPU" primaryQuery: 'sum(rate(process_cpu_seconds_total{component=~".*tikv"}[$rate])) by (cluster_id)' memResource: - name: "TiDB_MEM_P90" primaryQuery: 'quantile(0.9,sum(process_resident_memory_bytes{component="tidb"})by (instance,cluster))by (cluster)' - name: "TiDB_MEM_P80" primaryQuery: 'quantile(0.8,sum(process_resident_memory_bytes{component="tidb"})by (instance,cluster))by (cluster)' - name: "TiDB_MEM" primaryQuery: 'sum(process_resident_memory_bytes{component="tidb"})by (cluster)' - name: "TiKV_MEM_P90" primaryQuery: 'quantile(0.90,avg(process_resident_memory_bytes{component=~".*tikv"}) by (instance))by (cluster_id)' - name: "TiKV_MEM_P80" primaryQuery: 'quantile(0.80,avg(process_resident_memory_bytes{component=~".*tikv"}) by (instance))by (cluster_id)' - name: "TiKV_MEM" primaryQuery: 'sum(avg(process_resident_memory_bytes{component=~".*tikv"})by(instance))by(cluster_id)' latencyQueries: - name: "P99QueryLatency" primaryQuery: "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[$rate])) by (le))" - name: "P999QueryLatency" primaryQuery: "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[$rate])) by (le))" - name: "P99TiKVLatency" primaryQuery: 'histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type=~"coprocessor|kv_prewrite"}[$rate])) by (le,type))' - name: "CPS" primaryQuery: 'sum(rate(tidb_server_query_total{result="OK"}[$rate]))' - name: "QPS" primaryQuery: "sum(rate(tidb_executor_statement_total[$rate]))" diskQueries: - name: "ReadWaitLatency" primaryQuery: 'sum ( rate(node_disk_read_time_seconds_total{device!~"dm-.*|nvme1.*|nvme0.*"}[$rate]) / rate(node_disk_reads_completed_total{device!~"dm-.*|nvme1.*|nvme0.*"}[$rate]) )' - name: "WriteWaitLatency" primaryQuery: 'sum( rate(node_disk_write_time_seconds_total{device!~"dm-.*|nvme1.*|nvme0.*"}[$rate]) / rate(node_disk_writes_completed_total{device!~"dm-.*|nvme1.*|nvme0.*"}[$rate]) )' - name: "DiskUsage" primaryQuery: '(max((sum by (instance) (tikv_store_size_bytes{type="used"}) / sum by (instance) (tikv_store_size_bytes{type="capacity"}) * 100)))'