groups: - name: swat-kernel-governance rules: # - alert: WetechQPSRaised # annotations: # message: > # Wetech Cluster QPS last 7 days Raised more than 20%, need be scaled-out # expr: | # (avg_over_time(sum by (cluster_id) (rate(tidb_server_query_total{tenant=~"1372813089209061633|.*1372813089209289635", project!~"1372813089454536384"}[1d]))[7d:]) / avg_over_time(sum by (cluster_id) (rate(tidb_server_query_total{tenant=~"1372813089209061633|.*1372813089209289635", project!~"1372813089454536384"}[1d]))[14d:])) > 1.2 # for: 60s # labels: # severity: major # component: resilience # stability_governance: capacity - alert: TiDBCPUCoreUsageOver75ForBig3 annotations: description: 'TiDB single node CPU usage exceeds 75% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}, nodegroup: {{ $labels.cluster }}' summary: 'TiDB single node CPU usage over 75%' expr: | 100 * max by (tenant, cluster_id, cluster) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[1m])) by (tenant, cluster_id, cluster, instance) / sum(rate(node_cpu_seconds_total{ component=~"tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[1m])) by (tenant, cluster_id, cluster, instance) ) > 75 for: 5m labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance - alert: TiDBProcessCPUFrequentlyOver70ForBig3 annotations: description: 'TiDB instance process CPU usage has exceeded 70% of maxprocs for {{ $value }} minutes out of the last 20 minutes (Alert threshold is > 3 minutes). cluster_id: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }} mins (threshold: >3 mins) over 70% in last 20m' summary: 'TiDB single node process CPU usage frequently over 70%' expr: | sum_over_time( ( ( sum by (tenant, cluster_id, cluster, instance) ( irate(process_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[2m]) ) / sum by (tenant, cluster_id, cluster, instance) ( tidb_server_maxprocs{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } ) ) * 100 > bool 70 )[20m:1m] ) > 3 labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance - alert: TiKVCPUCoreUsageOver80ForBig3 annotations: description: 'TiKV single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiKV single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*10355969091959802668" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*10355969091959802668" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-governance - alert: TiKVCPUCoreUsageOver90ForBingoC66 annotations: description: 'TiKV single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiKV single node CPU usage over 90%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tikv", tenant=~".*1372813089209238420", project!~"1372813089454561820", cluster_id=~".*10355969091959802668" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"tikv", tenant=~".*1372813089209238420", project!~"1372813089454561820", cluster_id=~".*10355969091959802668" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 20m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-governance - alert: TiFlashCPUCoreUsageOver80ForBig3 annotations: description: 'TiFlash single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiFlash single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tiflash", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"tiflash", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: kernel-governance - alert: PDCPUCoreUsageOver80ForBig3 annotations: description: 'PD single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'PD single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"pd", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"pd", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: kernel-governance - alert: HighCPUImbalanceOnTiDB expr: | ( ( max by (cluster_id, cluster) ( ( 100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id, cluster, instance) ) / ( sum(rate(node_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id, cluster, instance) ) ) / ( ( 100 * avg( sum(rate(node_cpu_seconds_total{ mode!="idle", component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id, cluster) ) ) / ( avg( sum(rate(node_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id, cluster) ) ) ) ) > 1.5 ) and ( ( max by (cluster_id, cluster) ( ( 100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id, cluster, instance) ) / ( sum(rate(node_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id, cluster, instance) ) ) ) > 80 ) for: 5m labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: summary: "TiDB instance CPU usage is significantly higher than cluster average" description: "One TiDB node's CPU usage exceeds 80% and is 1.5x higher than the cluster's 15-minute average. This may indicate workload imbalance, SQL hotspots, or connection skew. Please check whether it is ddl leader / analyze owner / gc owner node." - alert: TiKVCPUImbalanceHighFor5m expr: | ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) / (sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) )) / ((100 * avg(sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id)) / avg(sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id))))) > 1.5 and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) / (sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) ))>65 for: 5m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-governance annotations: summary: "TiKV instance CPU usage is significantly higher than cluster average" description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV nodes continuously for 5 minutes. One TiKV node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}' - alert: TiKVMemoryImbalanceHighFor5m expr: | ( max by (cluster_id) ( 100 * (1 - ( node_memory_MemAvailable_bytes{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } / node_memory_MemTotal_bytes{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } )) ) / avg by (cluster_id) ( 100 * (1 - ( node_memory_MemAvailable_bytes{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } / node_memory_MemTotal_bytes{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } )) ) ) > 1.1 and max by (cluster_id) ( 100 * (1 - ( node_memory_MemAvailable_bytes{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } / node_memory_MemTotal_bytes{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" } )) ) > 81 for: 1m labels: severity: warning component: tikv oncall_team: tikv_unified stability_governance: kernel-governance annotations: summary: "TiKV instance memory usage is significantly higher than cluster average" description: 'Cluster {{ $labels.cluster_id }} has experienced memory load imbalance on TiKV nodes continuously for 1 minutes. One TiKV node''s memory usage exceeds 81% and is 1.08x higher than the cluster''s average memory usage. Value: {{ $value }}' - alert: TiKVCPUImbalanceHighFor20m expr: | ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) / (sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) )) / ((100 * avg(sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id)) / avg(sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id))))) > 1.5 and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) / (sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820" }[5m])) by (cluster_id,instance)) ))>85 for: 20m labels: severity: critical response_class: page component: tikv oncall_team: tikv_unified stability_governance: kernel-governance annotations: summary: "TiKV instance CPU usage is significantly higher than cluster average" description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV nodes continuously for 20 minutes. One TiKV node''s CPU usage exceeds 85% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}' # - alert: HighCPUImbalanceOnTiFlash # expr: | # ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ # mode!="idle", # component="tiflash", # tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", # project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) # / # (sum(rate(node_cpu_seconds_total{ # component="tiflash", # tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", # project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) )) # / # ((100 * avg(sum(rate(node_cpu_seconds_total{ # mode!="idle", # component="tiflash", # tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", # project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id)) # / # avg(sum(rate(node_cpu_seconds_total{ # component="tiflash", # tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", # project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id))))) > 1.5 # and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ # mode!="idle", # component="tiflash", # tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", # project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) # / # (sum(rate(node_cpu_seconds_total{ # component="tiflash", # tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", # project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) ))>50 # for: 5m # labels: # severity: major # component: tiflash # stability_governance: kernel-governance # annotations: # summary: "TiFlash instance CPU usage is significantly higher than cluster average" # description: "One TiFlash node's CPU usage exceeds 50% and is 1.5x higher than the cluster's 15-minute average." - alert: TiCDCCPUCoreUsageOver80ForBig3 annotations: description: 'TiCDC single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiCDC single node CPU usage over 80%' expr: | ( rate(process_cpu_seconds_total{ component="ticdc", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id=~".*10615475230796159612" }[2m]) / ticdc_server_go_max_procs{ component="ticdc", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id=~".*10615475230796159612" } ) * 100 > 80 for: 5m labels: severity: major response_class: ticket component: ticdc-kernel oncall_team: ticdc stability_governance: kernel-governance - alert: TiDBServerDowntimeOver30MinForBig3 expr: | up{ component=~"tidb", job=~".*tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" } == 0 for: 30m labels: tier: dedicated severity: critical response_class: page component: infra-provider oncall_team: dedicated_service stability_governance: kernel-governance annotations: description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.' summary: '{{ $labels.component }} server has been down for over 30 minutes' - alert: TiKVDowntimeOver30MinForBig3 expr: | up{ component=~"tikv", job=~".*tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" } == 0 for: 30m labels: tier: dedicated severity: critical response_class: page component: infra-provider oncall_team: dedicated_service stability_governance: kernel-governance annotations: description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.' summary: '{{ $labels.component }} server has been down for over 30 minutes' - alert: TiFlashDowntimeOver30MinForBig3 expr: | up{ component=~"tiflash", job=~".*tiflash", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" } == 0 for: 30m labels: tier: dedicated severity: critical response_class: page component: infra-provider oncall_team: dedicated_service stability_governance: kernel-governance annotations: description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.' summary: '{{ $labels.component }} server has been down for over 30 minutes' - alert: PDDowntimeOver30MinForBig3 expr: | up{ component=~"pd", job=~".*pd", tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" } == 0 for: 30m labels: tier: dedicated severity: critical response_class: page component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.' summary: '{{ $labels.component }} server has been down for over 30 minutes' - alert: TiCDCDowntimeOver30MinForBig3 expr: | up{ component=~"ticdc", job=~".*ticdc", tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" } == 0 for: 30m labels: tier: dedicated severity: critical response_class: page component: ticdc oncall_team: tidb_x_tools stability_governance: kernel-governance annotations: description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.' summary: '{{ $labels.component }} server has been down for over 30 minutes' - alert: TiKVMemoryCoreUsageOver85ForBig3 annotations: message: > TiKV node memory utilization over 85%. value: {{ $value }} source_tcoc: TCOC-2913 expr: |- max by(cluster_id)( 100 * (1 - ( node_memory_MemAvailable_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820",cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820",cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} )) )> 85 for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 # bingo 的三套集群包括 prod-dc Prod-Jili Prod-Others 都有类似的问题. 可能和 bingo 遇到的其他内存异常和 Bingoplus TiKV 内存问题调查记录 类似，目前不确定 root cause，如果后续周期性一直上涨明显接近 OOM 的话，只能先走 evict-leader + restart 来处理了. - alert: TiKVMemoryCoreUsageOver90ForBingo3Clusters annotations: message: > TiKV node memory utilization over 90%. value: {{ $value }} source_tcoc: TCOC-2913 expr: |- 100 * (1 - ( node_memory_MemAvailable_bytes{component="tikv", cluster_id=~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} / node_memory_MemTotal_bytes{component="tikv", cluster_id=~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} )) > 90 for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 - alert: TotalTiKVMemoryUsageOver85ForBig3 annotations: description: 'Total TiKV memory utilization over 85%. value: {{ $value }}' summary: 'Total TiKV memory utilization exceeds 85% for 10 minutes' expr: |- 1 - avg by (tidb_cluster_id) ( ( node_memory_MemFree_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} + node_memory_Buffers_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} + node_memory_Cached_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} ) / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} ) > 0.85 for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 - alert: TiKVCoreMemoryUsageOverAvgTotalForBig3 annotations: description: 'TiKV single node memory utilization over (Total TiKV node memory utilization + 5). value: {{ $value }} source_tcoc: TCOC-2913' summary: 'TiKV single node memory utilization exceeds the cluster average by 5 points' expr: |- (100 * (1 - ( node_memory_MemAvailable_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} ))) > (100 * (1 - avg by (tidb_cluster_id) ( ( node_memory_MemFree_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} + node_memory_Buffers_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} + node_memory_Cached_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} ) / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10264132336816574236|.*10890423899572033652|.*10594996726066304073"} ))+5) for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 - alert: TiDBP999LatencySurgeForBig3 expr: | histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612"}[1m])) by (le, cluster_id)) > 5 * histogram_quantile(0.999, avg_over_time( sum by (le, cluster_id) ( rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*1379661944646416096"}[5m]) )[24h:5m] )) for: 5m labels: tier: dedicated severity: critical response_class: page component: resilience oncall_team: fb_perf stability_governance: kernel-tcocp annotations: description: 'TiDB query P999 latency within the past 2 minutes exceeds 10 times the 24-hour P999 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb. cluster: {{ $labels.cluster_id }}, current 2-minutes P999 latency is {{ $value }}s, which exceeds 10x of the 24h P999 baseline' summary: TiDB query P999 latency significantly increased in short term # - alert: TiDBP999LatencySurgeForBingoMessage # expr: | # histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", cluster_id=~".*10615475230796159612"}[1m])) by (le, cluster_id)) # > # 35 * histogram_quantile(0.999, avg_over_time( # sum by (le, cluster_id) ( # rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb", cluster_id=~".*10615475230796159612"}[5m]) # )[24h:5m] # )) # for: 3m # labels: # tier: dedicated # severity: critical # component: resilience # stability_governance: kernel-tcocp # annotations: # description: 'cluster: {{ $labels.cluster_id }}, current 2-minutes P999 latency is {{ $value }}s, which exceeds 25x of the 24h P999 baseline' # summary: TiDB query P999 latency significantly increased in short term # message: > # TiDB query P999 latency within the past 2 minutes exceeds 25 times the 24-hour P999 baseline. # This may indicate query slowness, hotspots, resource contention, or downstream degradation. # SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb - alert: TiDBP95LatencySurgeForBig3 expr: | histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*10890423899572033652"}[1m])) by (le, cluster_id)) > 5 * histogram_quantile(0.95, avg_over_time( sum by (le, cluster_id) ( rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*10890423899572033652"}[5m]) )[24h:5m] )) for: 5m labels: tier: dedicated severity: critical response_class: page component: resilience oncall_team: fb_perf stability_governance: kernel-tcocp annotations: description: 'TiDB query P95 latency within the past 5 minutes exceeds 5 times the 24-hour P95 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb. cluster: {{ $labels.cluster_id }}, current 5-minutes P95 latency is {{ $value }}s, which exceeds 5x of the 24h P95 baseline' summary: TiDB query P95 latency significantly increased in short term - alert: TiDBP95LatencySurgeForBingoProdDC expr: | histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~"1372813089209238420", project!~"1372813089454561820", cluster_id=~".*10890423899572033652"}[1m])) by (le, cluster_id)) > 7 * histogram_quantile(0.95, avg_over_time( sum by (le, cluster_id) ( rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~"1372813089209238420", project!~"1372813089454561820", cluster_id=~".*10890423899572033652"}[5m]) )[24h:5m] )) for: 5m labels: tier: dedicated severity: critical response_class: page component: resilience oncall_team: fb_perf stability_governance: kernel-tcocp annotations: description: 'TiDB query P95 latency within the past 5 minutes exceeds 7 times the 24-hour P95 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb. cluster: {{ $labels.cluster_id }}, current 5-minutes P95 latency is {{ $value }}s, which exceeds 7x of the 24h P95 baseline' summary: TiDB query P95 latency significantly increased in short term - alert: RequestErrorRateOver10ForBig3 expr: |- ( sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}[5m])) by (cluster_id) / sum(increase(tidb_executor_statement_total{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}[5m])) by (cluster_id) ) > 0.1 and sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660" }[1m])) by (cluster_id) > 1000 for: 0s labels: severity: critical response_class: page component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: kernel-governance annotations: message: > Cluster {{ $labels.cluster_id }} the error rate is above 10%. Please check the recent errors and overall statement execution status for this cluster. - alert: RapidErrorRateIncreaseOver450ForBig3 expr: |- ( avg_over_time( ( sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384", cluster_id=~".*1379661944646414653|.*10218000100066887827", type="unknown"}[10m])) by (cluster_id) / sum(increase(tidb_executor_statement_total{tenant=~".*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384", cluster_id=~".*1379661944646414653|.*10218000100066887827"}[10m])) by (cluster_id) )[10m:1m] ) / avg_over_time( ( sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384", cluster_id=~".*1379661944646414653|.*10218000100066887827", type="unknown"}[1d])) by (cluster_id) / sum(increase(tidb_executor_statement_total{tenant=~".*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384", cluster_id=~".*1379661944646414653|.*10218000100066887827"}[1d])) by (cluster_id) )[1d:1h] ) ) > 4.5 and sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384", cluster_id=~".*1379661944646414653|.*10218000100066887827", type="unknown"}[10m])) by (cluster_id) > 1000 for: 0s labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: message: > Cluster {{ $labels.cluster_id }} The error rate in the last 1 minute is more than 4.5 times higher than the 5-minute average. - alert: TiDBNodeRestartForWeTechTest expr: |- time() - process_start_time_seconds{ tenant=~".*1372813089209061633|.*1372813089209289635",project!~"1372813089454536384",cluster_id=~".*10957523948617165785|.*10956278120794001035",component="tidb"}<60 for: 0s labels: severity: major response_class: ticket component: kubernetes oncall_team: dedicated_service stability_governance: kernel-governance annotations: message: > TiDB node restart occurred in cluster {{ $labels.cluster_id }} - alert: QPSDroppedBy80% expr: |- ( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618",project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id) ) < 0.2 * ( avg_over_time( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618",project!~"1372813089454536384|1372813089454561820" , cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id)[2h:] ) ) and ( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618",project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id) ) < 0.2 * ( avg_over_time( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618",project!~"1372813089454536384|1372813089454561820" , cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id)[1h:] ) ) and ( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618",project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id) ) > 1000 for: 5m labels: severity: critical response_class: page component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: message: > QPS Dropped By 80% in cluster {{ $labels.cluster_id }} - alert: QPSDroppedBy50% expr: |- ( sum by (cluster_id) ( rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]) ) ) < 0.50 * min by (cluster_id) ( label_replace( avg_over_time( ( sum by (cluster_id) ( rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]) ) )[1h:] ), "baseline", "recent_1h", "cluster_id", ".*" ) or label_replace( avg_over_time( ( sum by (cluster_id) ( rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]) ) )[2h:] ), "baseline", "recent_2h", "cluster_id", ".*" ) or label_replace( avg_over_time( ( sum by (cluster_id) ( rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m] offset 1d) ) )[30m:] ), "baseline", "yesterday_same_time", "cluster_id", ".*" ) ) and ( sum by (cluster_id) ( rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]) ) ) > 1000 for: 5m labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: message: > QPS Dropped By 50% in cluster {{ $labels.cluster_id }} - alert: QPSDroppedBy20%andTiDBP95LatencySurgeForBig3 expr: |- histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*10890423899572033652"}[1m])) by (le, cluster_id)) > 2 * histogram_quantile(0.95, avg_over_time( sum by (le, cluster_id) ( rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820", cluster_id!~".*10615475230796159612|.*10890423899572033652"}[5m]) )[24h:5m] )) and ( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635",project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id) ) < 0.8 * ( avg_over_time( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635",project!~"1372813089454536384|1372813089454561820" , cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id)[2h:] ) ) and ( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635",project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id) ) < 0.80 * ( avg_over_time( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635",project!~"1372813089454536384|1372813089454561820" , cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id)[1h:] ) ) and ( sum(rate(tidb_executor_statement_total{ tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635",project!~"1372813089454536384|1372813089454561820", cluster_id!~"tidb10615475230796159612" }[2m]))by (cluster_id) ) > 1000 for: 5m labels: severity: warning component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: message: > QPS Dropped By 20% And P95 Latency S in cluster {{ $labels.cluster_id }} And TiDB query P95 latency within the past 5 minutes exceeds 5 times the 24-hour P95 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb - alert: TiDBHighSpace75Over10mForBig3 expr: | label_replace( max by(cluster_id)( (1 - ( node_memory_MemAvailable_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",component="tidb"} / node_memory_MemTotal_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",component="tidb"} )) ) * 100 > 75, "cluster_id", "$1", "cluster_id", "^tidb(.*)$" ) for: 10m labels: severity: critical response_class: page component: tidb-executor oncall_team: tidb_executor stability_governance: kernel-tcocp annotations: message: > TiDB node memory usage is over 75% for more than 10 minutes in cluster {{ $labels.cluster_id }}. Current value: {{ $value }}% - alert: TiDBHighSpace70Over10mForBig3 expr: | label_replace( max by(cluster_id)( (1 - ( node_memory_MemAvailable_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",component="tidb"} / node_memory_MemTotal_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",component="tidb"} )) ) * 100 > 70, "cluster_id", "$1", "cluster_id", "^tidb(.*)$" ) for: 10m labels: severity: major response_class: ticket component: tidb-executor oncall_team: tidb_executor stability_governance: kernel-tcocp annotations: message: > TiDB node memory usage is over 70% for more than 10 minutes in cluster {{ $labels.cluster_id }}. Current value: {{ $value }}% - alert: TiCDCLogPullerMemoryQuotaHighWithCluster expr: |- sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~"$k8s_cluster", tidb_cluster_id="$tidb_cluster", instance=~"$ticdc_instance", module=~"log-puller", tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}) by (instance, type) > 100 for: 5m labels: severity: major response_class: ticket component: ticdc oncall_team: tidb_x_tools stability_governance: kernel-governance annotations: message: > TiCDC log puller memory quota is high in cluster {{ $labels.cluster_id }}. Current value: {{ $value }}%