groups: - name: swat-kernel-ue rules: # Special rules for UE - alert: QPSRaisedForUE annotations: message: > UE Cluster QPS last 7 days Raised more than 50%, need be scaled-out expr: | (avg_over_time(sum by (cluster_id) (rate(tidb_server_query_total{tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"}[1d]))[7d:]) / avg_over_time(sum by (cluster_id) (rate(tidb_server_query_total{tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"}[1d]))[14d:])) > 1.5 for: 60s labels: severity: major response_class: ticket component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: capacity - alert: TiDBCPUCoreUsageOver80ForUE annotations: description: 'TiDB single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}, nodegroup: {{ $labels.cluster }}' summary: 'TiDB single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id, cluster) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, cluster, instance) / sum(rate(node_cpu_seconds_total{ component=~"tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, cluster, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance - alert: TiKVCPUCoreUsageOver80ForUE annotations: description: 'TiKV single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiKV single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-governance - alert: TiFlashCPUCoreUsageOver80ForUE annotations: description: 'TiFlash single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiFlash single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"tiflash", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"tiflash", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: kernel-governance - alert: PDCPUCoreUsageOver80ForUE annotations: description: 'PD single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'PD single node CPU usage over 80%' expr: | 100 * max by (tenant, cluster_id) ( sum(rate(node_cpu_seconds_total{ mode!="idle", component=~"pd", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, instance) / sum(rate(node_cpu_seconds_total{ component=~"pd", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m])) by (tenant, cluster_id, instance) ) > 80 for: 5m labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: kernel-governance - alert: HighCPUImbalanceOnTiDBForUE expr: | ( ( max by (cluster_id, cluster) ( ( 100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id, cluster, instance) ) / ( sum(rate(node_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id, cluster, instance) ) ) / ( ( 100 * avg( sum(rate(node_cpu_seconds_total{ mode!="idle", component="tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id, cluster) ) ) / ( avg( sum(rate(node_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id, cluster) ) ) ) ) > 1.5 ) and ( ( max by (cluster_id, cluster) ( ( 100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id, cluster, instance) ) / ( sum(rate(node_cpu_seconds_total{ component="tidb", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id, cluster, instance) ) ) ) > 80 ) for: 5m labels: severity: major response_class: ticket component: resilience oncall_team: resilience stability_governance: kernel-governance annotations: summary: "TiDB instance CPU usage is significantly higher than cluster average" description: "One TiDB node's CPU usage exceeds 80% and is 1.5x higher than the cluster's 15-minute average. This may indicate workload imbalance, SQL hotspots, or connection skew. Please check whether it is ddl leader / analyze owner / gc owner node." - alert: HighCPUImbalanceOnTiKVForUE expr: | ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id,instance)) / (sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id,instance)) )) / ((100 * avg(sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id)) / avg(sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id))))) > 1.5 and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ mode!="idle", component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id,instance)) / (sum(rate(node_cpu_seconds_total{ component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[5m])) by (cluster_id,instance)) ))>65 for: 5m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-governance annotations: summary: "TiKV instance CPU usage is significantly higher than cluster average" description: "One TiKV node's CPU usage exceeds 65% and is 1.5x higher than the cluster's 15-minute average." # - alert: HighCPUImbalanceOnTiFlashForUE # expr: | # ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ # mode!="idle", # component="tiflash", # tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) # / # (sum(rate(node_cpu_seconds_total{ # component="tiflash", # tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) )) # / # ((100 * avg(sum(rate(node_cpu_seconds_total{ # mode!="idle", # component="tiflash", # tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id)) # / # avg(sum(rate(node_cpu_seconds_total{ # component="tiflash", # tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id))))) > 1.5 # and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{ # mode!="idle", # component="tiflash", # tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) # / # (sum(rate(node_cpu_seconds_total{ # component="tiflash", # tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660" # }[5m])) by (cluster_id,instance)) ))>50 # for: 5m # labels: # severity: major # component: tiflash # stability_governance: kernel-governance # annotations: # summary: "TiFlash instance CPU usage is significantly higher than cluster average" # description: "One TiFlash node's CPU usage exceeds 50% and is 1.5x higher than the cluster's 15-minute average." - alert: TiCDCCPUCoreUsageOver80ForUE annotations: description: 'TiCDC single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}' summary: 'TiCDC single node CPU usage over 80%' expr: | ( rate(process_cpu_seconds_total{ component="ticdc", tenant =~ "1372813089209256793", project!~"1372813089454576822|1372813089454574660" }[2m]) / ticdc_server_go_max_procs{ component="ticdc", tenant =~ "1372813089209256793", project!~"1372813089454576822|1372813089454574660" } ) * 100 > 80 for: 5m labels: severity: major response_class: ticket component: ticdc-kernel oncall_team: ticdc stability_governance: kernel-governance - alert: TiKVMemoryCoreUsageOver85ForUE annotations: message: > TiKV node memory utilization over 85%. source_tcoc: TCOC-2913 expr: |- max by(cluster_id)( 100 * (1 - ( node_memory_MemAvailable_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} )) ) > 85 for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 - alert: TotalTiKVMemoryUsageOver80ForUE annotations: description: 'Total TiKV memory utilization over 80% for 10 minutes' summary: 'Total TiKV memory utilization exceeds 80% for 10 minutes' expr: |- 1 - avg by (tidb_cluster_id) ( ( node_memory_MemFree_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660", cluster_id!~".*10408003622347968695|.*10665115838626244426"} + node_memory_Buffers_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660", cluster_id!~".*10408003622347968695|.*10665115838626244426"} + node_memory_Cached_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660", cluster_id!~".*10408003622347968695|.*10665115838626244426"} ) / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660", cluster_id!~".*10408003622347968695|.*10665115838626244426"} ) > 0.8 for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 - alert: TiKVCoreMemoryUsageOverAvgTotalForUE annotations: description: 'TiKV single node memory utilization over (Total TiKV node memory utilization + 5) source_tcoc: TCOC-2913' summary: 'TiKV single node memory utilization exceeds the cluster average by 5 points' expr: |- (100 * (1 - ( node_memory_MemAvailable_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} ))) > (100 * (1 - avg by (tidb_cluster_id) ( ( node_memory_MemFree_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} + node_memory_Buffers_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} + node_memory_Cached_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} ) / node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660"} ))+5) for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913 - alert: TiDBP95LatencySurgeForUE expr: | histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660",cluster_id!~".*10970866242657397869|.*10665115838626244426"}[5m])) by (le, cluster_id)) > 2 * histogram_quantile(0.95, avg_over_time( sum by (le, cluster_id) ( rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660",cluster_id!~".*10970866242657397869|.*10665115838626244426"}[5m]) )[24h:5m] )) for: 5m labels: tier: dedicated severity: critical response_class: page component: resilience oncall_team: resilience stability_governance: kernel-tcocp annotations: description: 'TiDB query P95 latency within the past 5 minute exceeds 2 times the 24-hour P95 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb. cluster: {{ $labels.cluster_id }}, current 5-minute P95 latency is {{ $value }}s, which exceeds 2x of the 24h P95 baseline' summary: TiDB query P95 latency significantly increased in short term - alert: TiDBP95LatencySurgeForUEDexAndChainTiDB expr: | histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660",cluster_id=~".*10970866242657397869|.*10665115838626244426"}[5m])) by (le, cluster_id)) > 5 * histogram_quantile(0.95, avg_over_time( sum by (le, cluster_id) ( rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~".*1372813089209256793", project!~"1372813089454576822|1372813089454574660",cluster_id=~".*10970866242657397869|.*10665115838626244426"}[5m]) )[24h:5m] )) for: 5m labels: tier: dedicated severity: critical response_class: page component: resilience oncall_team: resilience stability_governance: kernel-tcocp annotations: description: 'TiDB query P95 latency within the past 5 minute exceeds 5 times the 24-hour P95 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb. cluster: {{ $labels.cluster_id }}, current 5-minute P95 latency is {{ $value }}s, which exceeds 5x of the 24h P95 baseline' summary: TiDB query P95 latency significantly increased in short term - alert: TotalTiKVMemoryUsageOver85ForUEChain annotations: description: 'Total TiKV memory utilization over 85% for 10 minutes' summary: 'Total TiKV memory utilization exceeds 85% for 10 minutes' expr: |- 1 - avg by (tidb_cluster_id) ( ( node_memory_MemFree_bytes{component="tikv", cluster_id=~".*10408003622347968695|.*10665115838626244426"} + node_memory_Buffers_bytes{component="tikv", cluster_id=~".*10408003622347968695|.*10665115838626244426"} + node_memory_Cached_bytes{component="tikv", cluster_id=~".*10408003622347968695|.*10665115838626244426"} ) / node_memory_MemTotal_bytes{component="tikv", cluster_id=~".*10408003622347968695|.*10665115838626244426"} ) > 0.85 for: 10m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: kernel-tcocp source_tcoc: TCOC-2913