apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
labels:
observability.pingcap.com/o11y-overlay: "true"
name: benchmark
namespace: default
spec:
groups:
- interval: 15s
name: data-plane/tidb.rules
rules:
- expr: sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (cluster_id,le)
record: tidb_server_handle_query_duration_seconds_bucket:le:sum:rate
- expr: sum(rate(tidb_server_handle_query_duration_seconds_bucket{sql_type!="internal"}[2m]))
by (cluster_id,le)
record: tidb_server_handle_query_duration_seconds_bucket:le_sqltype_not:sum:rate
- expr: sum(rate(tidb_server_handle_query_duration_seconds_bucket{sql_type!="internal"}[2m]))
by (cluster_id,instance,le)
record: tidb_server_handle_query_duration_seconds_bucket:instance_le_sqltype_not:sum:rate
- expr: sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (cluster_id,le,sql_type)
record: tidb_server_handle_query_duration_seconds_bucket:le_sqltype:sum:rate
- expr: sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (cluster_id,instance,le)
record: tidb_server_handle_query_duration_seconds_bucket:le_instance:sum:rate
- expr: sum(rate(tidb_server_handle_query_duration_seconds_bucket{}[2m])) by (cluster_id,instance,sql_type,le)
record: tidb_server_handle_query_duration_seconds_bucket:sqltype_instance_le:sum:rate
- expr: sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='0'}[2m]))
by (cluster_id,le,in_txn)
record: tidb_server_conn_idle_duration_seconds_bucket:in_txn_zero:sum:rate
- expr: sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='1'}[2m]))
by (cluster_id,le,in_txn)
record: tidb_server_conn_idle_duration_seconds_bucket:in_txn_one:sum:rate
- expr: sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='0'}[2m]))
by (cluster_id,instance,le,in_txn)
record: tidb_server_conn_idle_duration_seconds_bucket:instance_in_txn_zero:sum:rate
- expr: sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='1'}[2m]))
by (cluster_id,instance,le,in_txn)
record: tidb_server_conn_idle_duration_seconds_bucket:instance_in_txn_one:sum:rate
- expr: sum(rate(tidb_session_transaction_duration_seconds_bucket{}[2m])) by (cluster_id,le,
txn_mode)
record: tidb_session_transaction_duration_seconds_bucket:le_txn_mode:sum:rate
- expr: sum(rate(tidb_session_transaction_duration_seconds_bucket{}[2m])) by (cluster_id,le,
txn_mode, instance)
record: tidb_session_transaction_duration_seconds_bucket:le_txn_mode_instance:sum:rate
- expr: sum(rate(tidb_session_transaction_duration_seconds_bucket{}[2m])) by (cluster_id,le,sql_type)
record: tidb_session_transaction_duration_seconds_bucket:le_sql_type:sum:rate
- expr: sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{}[2m])) by (cluster_id,le,type)
record: tidb_tikvclient_txn_cmd_duration_seconds_bucket:sum:rate
- name: data-plane/tidb
rules:
- alert: TiDBServerIsDown
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}} is down for 10 mins'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.tfuldltfya4a
summary: TiDB server is down
expr: up{component="tidb", job=~".*tidb"} == 0
for: 10m
labels:
component: tidb
expr: up{component="tidb", job=~".*tidb"} == 0
severity: major
tier: dedicated
- alert: TiDBServerIsDown
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}} is down for 5 mins'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.tfuldltfya4a
summary: TiDB server is down
expr: up{component="tidb", job=~".*tidb"} == 0
for: 5m
labels:
component: tidb
expr: up{component="tidb", job=~".*tidb"} == 0
severity: warning
tier: dedicated
- alert: TiDBDiscoveredTimeJumpBack
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ph7vof1otr6g
summary: TiDB monitor found time jump back error
expr: increase(tidb_monitor_time_jump_back_total[10m]) > 0
for: 5m
labels:
component: tidb
expr: increase(tidb_monitor_time_jump_back_total[10m]) > 0
severity: warning
tier: dedicated
- alert: TiDBHighTokenUsage
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}} token usage is values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.es82zlkd14z
summary: TiDB token usage too high
value: '{{ $value }}'
expr: sum(tidb_config_status{type="token-limit"}) by (cluster_id, instance,
tenant,provider_type) > 0 and (sum(tidb_server_tokens{}) by (cluster_id, instance,
tenant,provider_type) / sum(tidb_config_status{type="token-limit"}) by (cluster_id,
instance, tenant,provider_type)) > 0.8
for: 5m
labels:
component: tidb
expr: sum(tidb_config_status{type="token-limit"}) by (cluster_id, instance,
tenant,provider_type) > 0 and (sum(tidb_server_tokens{}) by (cluster_id,
instance, tenant,provider_type) / sum(tidb_config_status{type="token-limit"})
by (cluster_id, instance, tenant,provider_type)) > 0.8
severity: warning
tier: dedicated
- alert: TiDBQueryUnexpectedlyFailed
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, error type: {{ $labels.type }}, count: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.vr9n074q793p
summary: TiDB query got unexpected error
expr: increase(tidb_server_execute_error_total{type=~".*8027|.*8120|.*8021|.*8024|.*9001|.*9002|.*9003|.*9011|.*9012|.*9013"}[5m])
> 1
for: 5m
labels:
component: tidb
expr: increase(tidb_server_execute_error_total{type=~".*8027|.*8120|.*8021|.*8024|.*9001|.*9002|.*9003|.*9011|.*9012|.*9013"}[5m])
> 1
severity: warning
tier: dedicated
- alert: TiDBFrequentlyUnavailable
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, value: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.u233t02connq
summary: TiDB server unavailable time more than 3 minute in 10 minutes
expr: sum(changes(process_start_time_seconds{job=~".*tidb"}[1h])) by (cluster_id,
instance, tenant,provider_type) >=4
for: 1m
labels:
component: tidb
expr: sum(changes(process_start_time_seconds{job=~".*tidb"}[1h])) by (cluster_id,
instance, tenant,provider_type) >=4
severity: critical
tier: dedicated
- alert: TiDBStatsHealthy
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, value: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.k4nn0mporild
summary: More than half of tables's statisitics TiDB server are in bad healthy
for one day
expr: avg(tidb_statistics_stats_healthy{type="[0,50)"}) by (cluster_id, instance,
tenant, provider_type) / avg(tidb_statistics_stats_healthy{type="[0,100]"})
by (cluster_id, instance, tenant, provider_type) > 0.5
for: 1d
labels:
component: tidb
expr: avg(tidb_statistics_stats_healthy{type="[0,50)"}) by (cluster_id, instance,
tenant, provider_type) / avg(tidb_statistics_stats_healthy{type="[0,100]"})
by (cluster_id, instance, tenant, provider_type) > 0.5
severity: warning
tier: dedicated
- name: pd/pd
rules:
- alert: PDServerIsDown
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.f5wwrkav7k5f
summary: PD server is down
value: '{{ $value }}'
expr: up{component="pd", job=~".*pd"} == 0
for: 5m
labels:
component: pd
expr: up{component="pd", job=~".*pd"} == 0
severity: major
tier: dedicated
- alert: PDDiscoverDownStore
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.zigyfcz6fd17
summary: PD found store is down
value: '{{ $value }}'
expr: (sum ( pd_cluster_status{type="store_down_count"} ) by (cluster_id, instance,
tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id,
instance, tenant, provider_type) > 0)
for: 5m
labels:
component: pd
expr: (sum ( pd_cluster_status{type="store_down_count"} ) by (cluster_id,
instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by
(cluster_id, instance, tenant, provider_type) > 0)
severity: critical
tier: dedicated
- alert: PDDiscoverLowSpaceStore
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t3cn3udx76m7
summary: PD found some store space is too low
value: '{{ $value }}'
expr: (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id,
instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id,
instance, tenant, provider_type) > 0)
for: 5m
labels:
component: pd
expr: (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id,
instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by
(cluster_id, instance, tenant, provider_type) > 0)
severity: warning
tier: dedicated
- alert: PDRegionDownPeerTooLong
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.2boq1zqje9s
summary: PD found some region has down peer too long duration
value: '{{ $value }}'
expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m]))
by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
for: 5m
labels:
component: pd
expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m]))
by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 *
2)
severity: warning
tier: dedicated
- alert: PDRegionMissPeerTooLong
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.my2gfezc47ow
summary: PD found some region has miss votor peer too long duration
value: '{{ $value }}'
expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m]))
by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
for: 5m
labels:
component: pd
expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m]))
by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 *
2)
severity: major
tier: dedicated
- alert: TiKVRegionPendingPeerTooLong
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.lizm9t5f8fpu
summary: TiKV found some region has pending peer too long duration
value: '{{ $value }}'
expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m]))
BY (le, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
for: 5m
labels:
component: pd
expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m]))
BY (le, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
severity: major
tier: dedicated
- name: data-plane/tiflash
rules:
- alert: TiFlashServerIsDown
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
summary: TiFlash server is down
value: '{{ $value }}'
expr: up{component="tiflash"} == 0
for: 5m
labels:
component: tiflash
expr: up{component="tiflash"} == 0
severity: critical
tier: dedicated
- alert: TiFlashLowSpace
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, value: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
summary: TiFlash store space used more than 80%
value: '{{ $value }}'
expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id,
instance, tenant,provider_type) / sum(tiflash_system_current_metric_StoreSizeCapacity)
by (cluster_id, instance, tenant,provider_type) * 100 < 20
for: 5m
labels:
component: tiflash
expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id,
instance, tenant,provider_type) / sum(tiflash_system_current_metric_StoreSizeCapacity)
by (cluster_id, instance, tenant,provider_type) * 100 < 20
severity: warning
tier: dedicated
- alert: TiFlashVeryLowSpace
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, value: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.skuuhxdpaymf
summary: TiFlash store space used more than 90%
value: '{{ $value }}'
expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id,
instance, tenant,provider_type) / sum(tiflash_system_current_metric_StoreSizeCapacity)
by (cluster_id, instance, tenant,provider_type) * 100 < 10
for: 5m
labels:
component: tiflash
expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id,
instance, tenant,provider_type) / sum(tiflash_system_current_metric_StoreSizeCapacity)
by (cluster_id, instance, tenant,provider_type) * 100 < 10
severity: major
tier: dedicated
- alert: TiFlashInodeUsageMoreThan90%
annotations:
description: 'inode usage more than 90%, cluster: {{ $labels.tidb_cluster
}}, instance: {{ $labels.instance }}, value: {{ $value }}'
runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
summary: TiFlash inode usage more than 90%
value: '{{ $value }}'
expr: |
kubelet_volume_stats_inodes_used{component="tiflash"}/kubelet_volume_stats_inodes{component="tiflash"} > 0.9
for: 5m
labels:
component: tiflash
severity: critical
tier: dedicated
- name: tikv/tikv
rules:
- alert: TiKVServerIsDown
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
summary: TiKV server is down
value: '{{ $value }}'
expr: up{component="tikv", job=~".*tikv"} == 0
for: 5m
labels:
component: tikv
expr: up{component="tikv", job=~".*tikv"} == 0
severity: major
tier: dedicated
- alert: TiKVServerFrequentlyDown
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
summary: TiKV server is down for at least 4 times in a hour
value: '{{ $value }}'
expr: sum(changes(process_start_time_seconds{job=~".*tikv"}[1h])) by (cluster_id,
instance, tenant,provider_type) >=4
for: 10m
labels:
component: tikv
expr: sum(changes(process_start_time_seconds{job=~".*tikv"}[1h])) by (cluster_id,
instance, tenant,provider_type) >=4
severity: critical
tier: dedicated
- alert: TiKVServerRestarted
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
summary: TiKV server is down for at least 1 times in a hour
value: '{{ $value }}'
expr: sum(changes(process_start_time_seconds{job=~".*tikv"}[1h])) by (cluster_id,
instance, tenant,provider_type) >=1
for: 10m
labels:
component: tikv
expr: sum(changes(process_start_time_seconds{job=~".*tikv"}[1h])) by (cluster_id,
instance, tenant,provider_type) >=1
severity: warning
tier: dedicated
- alert: TiKVServerDownMoreThan2
annotations:
description: 'cluster: {{ $labels.cluster_id }}, down tikv server count: {{
$value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
summary: TiKV server down count more than 2
value: '{{ $value }}'
expr: count(up{component="tikv", job=~".*tikv"} == 0) by (cluster_id,tenant,provider_type)
>= 2
for: 5m
labels:
component: tikv
expr: count(up{component="tikv", job=~".*tikv"} == 0) by (cluster_id,tenant,provider_type)
>= 2
severity: major
tier: dedicated
- alert: TiKVLowSpace
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, value: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.3tz23916jlht
summary: TiKV store space used more than 80%
value: '{{ $value }}'
expr: sum(tikv_store_size_bytes{type="available"}) by (cluster_id, instance,
tenant,provider_type) / sum(tikv_store_size_bytes{type="capacity"}) by (cluster_id,
instance, tenant,provider_type) * 100 < 20
for: 5m
labels:
component: tikv
expr: sum(tikv_store_size_bytes{type="available"}) by (cluster_id, instance,
tenant,provider_type) / sum(tikv_store_size_bytes{type="capacity"}) by
(cluster_id, instance, tenant,provider_type) * 100 < 20
severity: warning
tier: dedicated
- alert: TiKVVeryLowSpace
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, value: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.w4zl0kl6ny53
summary: TiKV store space used more than 90%
value: '{{ $value }}'
expr: sum(tikv_store_size_bytes{type="available"}) by (cluster_id, instance,
tenant,provider_type) / sum(tikv_store_size_bytes{type="capacity"}) by (cluster_id,
instance, tenant,provider_type) * 100 < 10
for: 5m
labels:
component: tikv
expr: sum(tikv_store_size_bytes{type="available"}) by (cluster_id, instance,
tenant,provider_type) / sum(tikv_store_size_bytes{type="capacity"}) by
(cluster_id, instance, tenant,provider_type) * 100 < 10
severity: major
tier: dedicated
- alert: TiKVRaftstoreCpuHighLoad
annotations:
description: |-
TiKV node's raftstore is under a high load
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ynohpbfagetz
summary: 'Raftstore high CPU load (cluster: {{ $labels.cluster_id }}, instance
{{ $labels.instance }})'
expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by
(cluster_id, instance, tenant,provider_type) / count(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"})
by (cluster_id, instance, tenant,provider_type) > 0.8
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVApplyWorkerPoolCpuHighLoad
annotations:
description: |-
TiKV node's apply worker thread pool is under a high load
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.7k1o575l3zea
summary: 'Apply worker thread pool high CPU load (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+"}[1m])) by
(cluster_id, instance, tenant,provider_type) / count(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+"})
by (cluster_id, instance, tenant,provider_type) > 0.9
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVSchedulerWorkerCpuHighLoad
annotations:
description: |-
TiKV node's scheduler worker thread pool is under a high load
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.qptrnffkw5h
summary: 'Scheduler worker high CPU load (cluster: {{ $labels.cluster_id }},
instance {{ $labels.instance }})'
expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"sched_.*"}[1m])) by (cluster_id,
instance, tenant,provider_type) / count(tikv_thread_cpu_seconds_total{name=~"sched_.*"})
by (cluster_id, instance, tenant,provider_type) > 0.9
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVGrpcPollHighLoad
annotations:
description: |-
TiKV node's gRPC polling thread pool is under a high load
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.30ra2mgpp1i4
summary: 'gRPC high CPU load (cluster: {{ $labels.cluster_id }}, instance
{{ $labels.instance }})'
expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"grpc.*"}[1m])) by (cluster_id,
instance, tenant,provider_type) / count(tikv_thread_cpu_seconds_total{name=~"grpc.*"})
by (cluster_id, instance, tenant,provider_type) > 0.8
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVUnifiedReadPoolHighLoad
annotations:
description: |-
TiKV node's unified read pool is under a high load
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.y1jc473vb50t
summary: 'Unified read pool high CPU load (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"unified_read_po*"}[1m]))
by (cluster_id, instance, tenant,provider_type) / count(tikv_thread_cpu_seconds_total{name=~"unified_read_po*"})
by (cluster_id, instance, tenant,provider_type) > 0.9
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVTooManyRaftLogDroppedCount
annotations:
description: |-
TiKV node's dropped too many raft log under high memory usage
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'Too many raft log dropped, likely due to excessive memory usage
(cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})'
expr: sum(rate(tikv_server_raft_append_rejects{type!="kv_gc"}[1m])) by (cluster_id,
instance, tenant, provider_type) > 100
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVRaftEntryCacheEvicts
annotations:
description: |-
TiKV node's Raft Entry Cache evicts at high speed, under high memory usage
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'Raft Entry Cache Evicts at high speed, likely due to excessive memory
usage (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})'
expr: rate(tikv_raft_entries_evict_bytes[1m]) > 5242880
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVTooManyFailedGRPCRequests
annotations:
description: |-
TiKV node's has too many grpc requests failed
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'TiKV node has too many grpc requests failed (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: rate(tikv_grpc_msg_fail_total{type!="kv_gc"}[1m]) > 100
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVRaftAppendLogSlow
annotations:
description: |-
TiKV node's raft append log P99 duration is too slow
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'TiKV node''s raft append log P99 duration in seconds is too slow
(cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})'
expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 30
for: 5m
labels:
component: tikv
level: major
tier: dedicated
- alert: TiKVRaftApplyLogDurationSecs
annotations:
description: |-
TiKV node's raft apply log P99 duration in seconds is too slow
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'TiKV node''s raft apply log P99 duration is too slow (cluster: {{
$labels.cluster_id }}, instance {{ $labels.instance }})'
expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 45
for: 5m
labels:
component: tikv
level: major
tier: dedicated
- alert: TiKVReportTooManyFailureMsg
annotations:
description: |-
TiKV node reported too many unreachable failure messages
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'TiKV node reported too many unreachable failure messages (cluster:
{{ $labels.cluster_id }}, instance {{ $labels.instance }})'
expr: sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[1m]))
BY (cluster_id, instance, tenant, provider_type) > 100
for: 5m
labels:
component: tikv
level: warning
tier: dedicated
- alert: TiKVChannelFull
annotations:
description: |-
TiKV node's gRPC channel is full
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'TiKV node''s gRPC channel is full (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: sum(rate(tikv_channel_full_total[1m])) BY (type, cluster_id, instance,
tenant, provider_type) > 0
for: 5m
labels:
component: tikv
level: warning
tier: dedicated
- alert: TiKVRaftLogLag
annotations:
description: |-
TiKV node's Raft log lap is more than 5000
VALUE = {{ $value }}
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
summary: 'TiKV node''s Raft log lap is more than 5000 (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: histogram_quantile(1, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by
(le, cluster_id, instance, tenant, provider_type)) > 5000
for: 1m
labels:
component: tikv
level: warning
tier: dedicated
- alert: TiKVSlowScoreTooHigh
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
summary: 'TiKV node reported a slow store (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: sum(tikv_raftstore_slow_score) by (cluster_id, instance, tenant, provider_type)
>= 80
for: 1m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVLargeLatencyNode
annotations:
description: 'cluster: {{ $labels.cluster_id }}, values: {{ $value }}'
runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
summary: 'TiKV node reported a slow TiKV node---its commit log duration is
5x more of the median value (cluster: {{ $labels.cluster_id }})'
expr: max(histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m]))
by (cluster_id, instance, tenant, provider_type))) >= 5 * quantile(0.5, histogram_quantile(0.99,
sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (cluster_id,
instance, tenant, provider_type)))
for: 5m
labels:
component: tikv
severity: warning
tier: dedicated
- alert: TiKVTooManyRegions
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, current region count: {{ $value }}'
summary: 'TiKV instance contains regions more than 160k (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: sum(tikv_raftstore_region_count{namespace=~".*1379661944636804070", type="region"})
by (cluster_id, instance, tenant, provider_type) > 160000
for: 10m
labels:
component: tikv
severity: critical
tier: dedicated
- alert: TiKVTooManyRegions
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, current region count: {{ $value }}'
summary: 'TiKV instance contains regions more than 130k (cluster: {{ $labels.cluster_id
}}, instance {{ $labels.instance }})'
expr: sum(tikv_raftstore_region_count{namespace=~".*1379661944636804070", type="region"})
by (cluster_id, instance, tenant, provider_type) > 130000
for: 10m
labels:
component: tikv
severity: major
tier: dedicated
- alert: TiKVInodeUsageMoreThan90%
annotations:
description: 'inode usage more than 90%, cluster: {{ $labels.tidb_cluster
}}, instance: {{ $labels.instance }}, value: {{ $value }}'
summary: TiKV inode usage more than 90%
value: '{{ $value }}'
expr: |
kubelet_volume_stats_inodes_used{component="tikv"}/kubelet_volume_stats_inodes{component="tikv"} > 0.9
for: 5m
labels:
component: tikv
severity: critical
tier: dedicated
- name: data-plane/backup-restore
rules:
- alert: LogBackupRunningRPOMoreThan10m
annotations:
message: The log backup task {{ $labels.task }} RPO of TiDB cluster {{ $labels.tenant
}}/{{ $labels.tidb_cluster }} is more than 10m
summary: RPO of log backup is high
expr: |
max(time() - tidb_log_backup_last_checkpoint / 262144000) by (tenant, tidb_cluster, task) / 60 > 10
and max(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, task) > 0
and max(tikv_log_backup_task_status) by (tenant, tidb_cluster, task) == 0
labels:
component: br
severity: warning
- alert: LogBackupRunningRPOMoreThan30m
annotations:
message: The log backup task {{ $labels.task }} RPO of TiDB cluster {{ $labels.tenant
}}/{{ $labels.tidb_cluster }} is more than {{ $value }}m
summary: RPO of log backup is too high
expr: |
max(time() - tidb_log_backup_last_checkpoint / 262144000) by (tenant, tidb_cluster, task) / 60 > 30
and max(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, task) > 0
and max(tikv_log_backup_task_status) by (tenant, tidb_cluster, task) == 0
labels:
component: br
severity: critical
- alert: LogBackupPausingMoreThan2h
annotations:
message: The log backup task {{ $labels.task }} of TiDB cluster {{ $labels.tenant
}}/{{ $labels.tidb_cluster }} paused {{ $value }}h
summary: Log backup paused more than 2h
expr: |
max(time() - tidb_log_backup_last_checkpoint / 262144000) by (tenant, tidb_cluster, task) / 3600 > 2
and max(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, task) > 0
and max(tikv_log_backup_task_status) by (tenant, tidb_cluster, task) == 1
labels:
component: br
severity: warning
- alert: LogBackupPausingMoreThan12h
annotations:
message: The log backup task {{ $labels.task }} of TiDB cluster {{ $labels.tenant
}}/{{ $labels.tidb_cluster }} paused {{ $value }}h
summary: Log backup paused more than 12h
expr: |
max(time() - tidb_log_backup_last_checkpoint / 262144000) by (tenant, tidb_cluster, task) / 3600 > 12
and max(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, task) > 0
and max(tikv_log_backup_task_status) by (tenant, tidb_cluster, task) == 1
labels:
component: br
severity: critical
- alert: LogBackupFailed
annotations:
message: The log backup task {{ $labels.task }} of TiDB cluster {{ $labels.tenant
}}/{{ $labels.tidb_cluster }} failed
summary: Log backup failed
expr: |
max(tikv_log_backup_task_status) by (tenant, tidb_cluster, task) == 2
and max(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, task) > 0
labels:
component: br
severity: critical
- alert: LogBackupGCSafePointExceedsCheckpoint
annotations:
message: The log backup of TiDB cluster {{ $labels.tenant }}/{{ $labels.tidb_cluster
}} gc safe point ts exceeds checkpoint ts
summary: Log backup gc safe point excceeds checkpoint
expr: |
min(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, instance) - max(tikv_gcworker_autogc_safe_point) by (tenant, tidb_cluster, instance) < 0
labels:
component: br
severity: critical
- name: data-plane/lightning
rules:
- alert: TiKVStoreSizeImbalanceDuringImport
annotations:
message: TiKV sizes of Tenant {{ $labels.tenant}}, Cluster {{ $labels.namespace
}} are imbalance during import for 5 longer than minutes.
summary: Sizes of TiKV stores are imbalance during import
expr: |
(
(stddev(sum(tikv_engine_size_bytes{cluster_id=~"tidb.*"}) by (instance, tenant, cluster_id)) by (tenant, cluster_id))
/
(avg(sum(tikv_engine_size_bytes{cluster_id=~"tidb.*"}) by (instance, tenant, cluster_id)) by (tenant, cluster_id))
# coefficient of variation
> 0.2
) and (
# lightning job is running
sum(lightning_engines{cluster_id=~"tidb.*"}) by (tenant, cluster_id)
)
for: 5m
labels:
component: lightning
severity: critical
- name: data-plane/ticdc
rules:
- alert: cdc_multiple_owners
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc cluster has multiple owners
value: '{{ $value }}'
expr: sum(rate(ticdc_owner_ownership_counter[30s])) by (cluster_id, instance,
tenant, provider_type) >= 2
for: 1m
labels:
component: ticdc
expr: sum(rate(ticdc_owner_ownership_counter[30s])) by (cluster_id, instance,
tenant, provider_type) >= 2
severity: warning
- alert: cdc_checkpoint_high_delay
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc processor checkpoint delay more than 10 minutes
value: '{{ $value }}'
expr: ticdc_owner_checkpoint_ts_lag{cluster_id!="tidb1379661944638124241"} >
600
for: 1m
labels:
component: ticdc
expr: ticdc_owner_checkpoint_ts_lag > 600
severity: critical
- alert: cdc_checkpoint_high_delay_for_60m
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc processor checkpoint delay more than 60 minutes
value: '{{ $value }}'
expr: ticdc_owner_checkpoint_ts_lag{cluster_id="tidb1379661944638124241"} >
3600
for: 1m
labels:
component: ticdc
expr: ticdc_owner_checkpoint_ts_lag > 3600
severity: critical
- alert: ticdc_puller_entry_sorter_sort_duration_time_more_than_2s
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: ticdc_puller_entry_sorter sort duration time more than 2s
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_sort_bucket[1m]))
by (le,cluster_id,instance,tenant, provider_type)) > 2
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_sort_bucket[1m]))
by (le,cluster_id,instance,tenant, provider_type)) > 2
severity: warning
- alert: ticdc_puller_entry_sorter_merge_duration_time_more_than_2s
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: ticdc_puller_entry_sorter merge duration time more than 2s
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_merge_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 2
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_merge_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 2
severity: warning
- alert: ticdc_mounter_unmarshal_and_mount_time_more_than_1s
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc_mounter unmarshal and mount time more than 1s
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) * 1000 > 1000
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(ticdc_mounter_unmarshal_and_mount_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) * 1000 > 1000
severity: warning
- alert: cdc_sink_execute_duration_time_more_than_10s
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc sink execute_duration_time_more_than_10s
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(ticdc_sink_txn_exec_duration_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 10
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(ticdc_sink_txn_exec_duration_bucket[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 10
severity: warning
- alert: cdc_owner_checkpoint_tso_no_change_for_1m
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc processor checkpoint tso no change for 1m
value: '{{ $value }}'
expr: changes(ticdc_owner_checkpoint_ts[1m]) < 1
labels:
component: ticdc
expr: changes(ticdc_owner_checkpoint_ts[1m]) < 1
severity: warning
- alert: ticdc_puller_entry_sorter_sort_bucket
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: ticdc puller entry sorter sort latency is too high
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_sort_bucket{}[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 1
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_sort_bucket{}[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 1
severity: warning
- alert: ticdc_puller_entry_sorter_merge_bucket
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: ticdc puller entry sorter merge latency is too high
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_merge_bucket{}[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 1
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(ticdc_puller_entry_sorter_merge_bucket{}[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 1
severity: warning
- alert: tikv_cdc_min_resolved_ts_no_change_for_1m
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: tikv cdc min resolved ts no change for 1m
value: '{{ $value }}'
expr: changes(tikv_cdc_min_resolved_ts[1m]) < 1
for: 1m
labels:
component: ticdc
expr: changes(tikv_cdc_min_resolved_ts[1m]) < 1
severity: warning
- alert: tikv_cdc_scan_duration_seconds_more_than_30s
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: tikv cdc scan duration seconds more than 30s
value: '{{ $value }}'
expr: histogram_quantile(0.9, sum(rate(tikv_cdc_scan_duration_seconds_bucket{}[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 30
for: 1m
labels:
component: ticdc
expr: histogram_quantile(0.9, sum(rate(tikv_cdc_scan_duration_seconds_bucket{}[1m]))
by (le, cluster_id, instance, tenant, provider_type)) > 30
severity: warning
- alert: ticdc_sink_mysql_execution_error
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc sink mysql execution meets errors
value: '{{ $value }}'
expr: changes(ticdc_sink_mysql_execution_error[1m]) > 0
for: 1m
labels:
component: ticdc
expr: changes(ticdc_sink_mysql_execution_error[1m]) > 0
severity: warning
- alert: ticdc_processor_exit_with_error_count
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values: {{ $value }}'
summary: cdc processor exits with error
value: '{{ $value }}'
expr: changes(ticdc_processor_exit_with_error_count[1m]) > 0
for: 1m
labels:
component: ticdc
expr: changes(ticdc_processor_exit_with_error_count[1m]) > 0
severity: critical
- alert: ticdc_memory_abnormal
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
summary: TiCDC heap memory usage is over 10 GB
value: '{{ $value }}'
expr: go_memstats_heap_alloc_bytes{job=~".*ticdc"} > 1e+10
for: 1m
labels:
component: ticdc
expr: go_memstats_heap_alloc_bytes{job=~".*ticdc"} > 1e+10
severity: warning
- alert: ticdc_open_fd_too_high
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
summary: TiCDC open file descriptors is too high
value: '{{ $value }}'
expr: process_open_fds{job=~".*ticdc"} / process_max_fds{job=~".*ticdc"} * 100
> 60
for: 5m
labels:
component: ticdc
expr: process_open_fds{job=~".*ticdc"} / process_max_fds{job=~".*ticdc"} *
100 > 60
severity: warning
- alert: ticdc_open_fd_too_high
annotations:
description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance
}}, values:{{ $value }}'
summary: TiCDC open file descriptors is too high
value: '{{ $value }}'
expr: process_open_fds{job=~".*ticdc"} / process_max_fds{job=~".*ticdc"} * 100
> 80
for: 5m
labels:
component: ticdc
expr: process_open_fds{job=~".*ticdc"} / process_max_fds{job=~".*ticdc"} *
100 > 80
severity: critical
- name: customer/data-migration
rules:
- alert: DMMetErrorOnExport
annotations:
description: Data migration job met error during data export
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName meet some error during data export. Please check the error and see Troubleshoot DM for help.
expr: |
changes(dm_mydumper_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(project_id, job_id, source_id) increase(dm_mydumper_exit_with_error_count{resumable_err="true"}[2m]) > 3
labels:
component: data-migration
description: Data migration job met error during data export
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName meet some error during data export. Please check the error and see Troubleshoot DM for help.
ruleID: "5"
severity: major
status: active
visibility: external
- alert: DMMetErrorOnImport
annotations:
description: Data migration job met error during data import
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName meet some error during data import. Please check the error and see Troubleshoot DM for help.
expr: |
changes(dm_loader_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(project_id, job_id, source_id) increase(dm_loader_exit_with_error_count{resumable_err="true"}[2m]) > 3
labels:
component: data-migration
description: Data migration job met error during data import
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName meet some error during data import. Please check the error and see Troubleshoot DM for help.
ruleID: "10"
severity: major
status: active
visibility: external
- alert: DMMetErrorOnSync
annotations:
description: Data migration job met error during incremental data migration
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName meet some error during incremental data migration. Please check the error and see Troubleshoot DM for help.
expr: |
changes(dm_syncer_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(project_id, job_id, source_id) increase(dm_syncer_exit_with_error_count{resumable_err="true"}[2m]) > 3
labels:
component: data-migration
description: Data migration job met error during incremental data migration
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName meet some error during incremental data migration. Please check the error and see Troubleshoot DM for help.
ruleID: "15"
severity: major
status: active
visibility: external
- alert: DMSyncPauseTooLong
annotations:
description: Data migration job has been paused for more than 6 hours during
incremental migration
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName has been paused for more than 6 hours during incremental migration, binlog in upstream database might be purged(depends on your database binlog purge strategy) and may cause incremental migration fail. See Troubleshoot DM for help.
expr: |
dm_worker_task_state{} == 3
for: 6h
labels:
component: data-migration
description: Data migration job has been paused for more than 6 hours during
incremental migration
message: |
Data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName has been paused for more than 6 hours during incremental migration, binlog in upstream database might be purged(depends on your database binlog purge strategy) and may cause incremental migration fail. See Troubleshoot DM for help.
ruleID: "20"
severity: major
status: active
visibility: external
- alert: DMReplicationLagTooLarge
annotations:
description: Replication lag is larger than {threshold} seconds and stilling
increasing for more than {triggerInterval} minutes
message: |
Replication lag is larger than {threshold} seconds and stilling increasing for more than {triggerInterval} minutes for data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName. See Troubleshoot DM for help.
expr: |
delta(dm_syncer_replication_lag_gauge[1m]) > 0 and on(project_id, job_id, source_id) dm_syncer_replication_lag_gauge{} > 600
for: 20m
labels:
canEdit: all
component: data-migration
description: Replication lag is larger than {threshold} seconds and stilling
increasing for more than {triggerInterval} minutes
message: |
Replication lag is larger than {threshold} seconds and stilling increasing for more than {triggerInterval} minutes for data migration job {{ $labels.job_id }} of TiDB cluster TiDBClusterName. See Troubleshoot DM for help.
monitorTarget: 'Replication lag '
operator: ' is larger than {threshold} seconds'
ruleID: "25"
severity: major
status: active
visibility: external
- name: customer/resources
rules:
- alert: TotalTiDBMemoryUsageTooHigh
annotations:
description: Total TiDB node memory utilization across cluster exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiDB node memory utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiDB to reduce the memory usage percentage of the current workload. To monitor node memory utilization, see Metrics page topic.
expr: 1 - avg by (tidb_cluster_id) (node_memory_MemAvailable_bytes{component="tidb"}
/ node_memory_MemTotal_bytes{component="tidb"}) > 0.7
for: 10m
labels:
canEdit: all
component: tidb
description: Total TiDB node memory utilization across cluster exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiDB node memory utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiDB to reduce the memory usage percentage of the current workload. To monitor node memory utilization, see Metrics page topic.
monitorTarget: 'Total TiDB node memory utilization '
operator: ' is greater than {thresholdPercentage}%'
ruleID: "10"
severity: major
status: active
visibility: external
- alert: TotalTiKVMemoryUsageTooHigh
annotations:
description: Total TiKV node memory utilization across cluster exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiKV node memory utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiKV to reduce the memory usage percentage of the current workload. To monitor node memory utilization, see Metrics page topic.
expr: 1 - avg by (tidb_cluster_id) (node_memory_MemAvailable_bytes{component="tikv"}
/ node_memory_MemTotal_bytes{component="tikv"}) > 0.7
for: 10m
labels:
canEdit: all
component: tikv
description: Total TiKV node memory utilization across cluster exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiKV node memory utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiKV to reduce the memory usage percentage of the current workload. To monitor node memory utilization, see Metrics page topic.
monitorTarget: Total TiKV node memory utilization
operator: ' is greater than {thresholdPercentage} %'
ruleID: "15"
severity: major
status: active
visibility: external
- alert: TotalTiFlashMemoryUsageTooHigh
annotations:
description: Total TiFlash node memory utilization across cluster exceeded
{thresholdPercentage}% for {triggerInterval} minutes
message: |
Total TiFlash node memory utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiKV to reduce the memory usage percentage of the current workload. To monitor node memory utilization, see Metrics page topic.
expr: 1 - avg by (tidb_cluster_id) (node_memory_MemAvailable_bytes{component="tiflash"}
/ node_memory_MemTotal_bytes{component="tiflash"}) > 0.7
for: 10m
labels:
canEdit: all
component: tiflash
description: Total TiFlash node memory utilization across cluster exceeded
{thresholdPercentage}% for {triggerInterval} minutes
message: |
Total TiFlash node memory utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiKV to reduce the memory usage percentage of the current workload. To monitor node memory utilization, see Metrics page topic.
monitorTarget: 'Total TiFlash node memory utilization '
operator: ' is greater than {thresholdPercentage} %'
ruleID: "20"
severity: major
status: active
visibility: external
- alert: TotalTiDBCPUUsageTooHigh
annotations:
description: Total TiDB node CPU utilization exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiDB node CPU utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiDB to reduce the CPU usage percentage of the current workload. To monitor node CPU utilization, see Metrics page topic.
expr: avg(sum by (cpu, instance, tidb_cluster_id) (irate(node_cpu_seconds_total{component="tidb",
mode!~"idle|iowait"}[2m]))) by (tidb_cluster_id) > 0.8
for: 10m
labels:
canEdit: all
component: tidb
description: Total TiDB node CPU utilization exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiDB node CPU utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiDB to reduce the CPU usage percentage of the current workload. To monitor node CPU utilization, see Metrics page topic.
monitorTarget: 'Total TiDB node CPU utilization '
operator: ' is greater than {thresholdPercentage}%'
ruleID: "30"
severity: major
status: active
visibility: external
- alert: TotalTiKVCPUUsageTooHigh
annotations:
description: Total TiKV node CPU utilization exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiKV node CPU utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiKV to reduce the CPU usage percentage of the current workload. To monitor node CPU utilization, see Metrics page topic.
expr: avg(sum by (cpu, instance, tidb_cluster_id) (irate(node_cpu_seconds_total{component="tikv",
mode!~"idle|iowait"}[2m]))) by (tidb_cluster_id) > 0.8
for: 10m
labels:
canEdit: all
component: tikv
description: Total TiKV node CPU utilization exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiKV node CPU utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiKV to reduce the CPU usage percentage of the current workload. To monitor node CPU utilization, see Metrics page topic.
monitorTarget: Total TiKV node CPU utilization
operator: ' is greater than {thresholdPercentage} %'
ruleID: "35"
severity: major
status: active
visibility: external
- alert: TotalTiFlashCPUUsageTooHigh
annotations:
description: Total TiFlash node CPU utilization exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiFlash node CPU utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiFlash to reduce the CPU usage percentage of the current workload. To monitor node CPU utilization, see Metrics page topic.
expr: avg(sum by (cpu, instance, tidb_cluster_id) (irate(node_cpu_seconds_total{component="tiflash",
mode!~"idle|iowait"}[2m]))) by (tidb_cluster_id) > 0.8
for: 10m
labels:
canEdit: all
component: tiflash
description: Total TiFlash node CPU utilization exceeded {thresholdPercentage}%
for {triggerInterval} minutes
message: |
Total TiFlash node CPU utilization of cluster TiDBClusterName in project TiDBProjectName has exceeded exceeded {thresholdPercentage}% for {triggerInterval} minutes. Consider increasing the node number or node size for TiFlash to reduce the CPU usage percentage of the current workload. To monitor node CPU utilization, see Metrics page topic.
monitorTarget: 'Total TiFlash node CPU utilization '
operator: ' is greater than {thresholdPercentage} %'
ruleID: "40"
severity: major
status: active
visibility: external
- alert: TiKVStorageUsageTooHigh
annotations:
description: TiKV storage utilization exceeds {thresholdPercentage}%
message: |
Total TiKV storage utilization of cluster TiDBClusterName in project TiDBProjectName exceeds {thresholdPercentage}%. Consider increasing the node number or node storage size for TiKV to increase your storage capacity. To monitor storage utilization, see Metrics page topic.
expr: sum(tikv_store_size_bytes{type="used"}) by (tidb_cluster_id) / sum (tikv_store_size_bytes{type="capacity"})
by (tidb_cluster_id) > 0.8
for: 10m
labels:
canEdit: all
component: tikv
description: TiKV storage utilization exceeds {thresholdPercentage}%
message: |
Total TiKV storage utilization of cluster TiDBClusterName in project TiDBProjectName exceeds {thresholdPercentage}%. Consider increasing the node number or node storage size for TiKV to increase your storage capacity. To monitor storage utilization, see Metrics page topic.
monitorTarget: 'Total TiKV storage utilization '
operator: ' is greater than {thresholdPercentage} %'
ruleID: "45"
severity: major
status: active
visibility: external
- alert: TiFlashStorageUsageTooHigh
annotations:
description: TiFlash storage utilization exceeds {thresholdPercentage}%
message: |
Total TiFlash storage utilization of cluster TiDBClusterName in project TiDBProjectName exceeds {thresholdPercentage}%. Consider increasing the node number or node storage size for TiFlash to increase your storage capacity. To monitor storage utilization, see Metrics page topic.
expr: sum(tiflash_system_current_metric_StoreSizeUsed{}) by (tidb_cluster_id)
/ sum(tiflash_system_current_metric_StoreSizeCapacity{}) by (tidb_cluster_id)
> 0.8
for: 10m
labels:
canEdit: all
component: tiflash
description: TiFlash storage utilization exceeds {thresholdPercentage}%
message: |
Total TiFlash storage utilization of cluster TiDBClusterName in project TiDBProjectName exceeds {thresholdPercentage}%. Consider increasing the node number or node storage size for TiFlash to increase your storage capacity. To monitor storage utilization, see Metrics page topic.
monitorTarget: 'Total TiFlash storage utilization '
operator: ' is greater than {thresholdPercentage} %'
ruleID: "50"
severity: major
status: active
visibility: external
- name: customer/ticdc
rules:
- alert: TiCDCCheckpointHighDelay
annotations:
description: Changefeed processor checkpoint delay more than {threshold} seconds
message: |
The Changefeed {{ $labels.changefeed }} of cluster {{ $labels.cluster_id }} checkpoint delay is more than {threshold} seconds. Please check if the downstream system and network configuration are normal, and also exclude the possibility of an indexed table. If the problem persists, please contact our support team directly for further assistance.
expr: ticdc_owner_checkpoint_ts_lag{} > 600
for: 1m
labels:
canEdit: all
component: ticdc
description: Changefeed processor checkpoint delay more than {threshold} seconds
message: |
The Changefeed {{ $labels.changefeed }} of cluster {{ $labels.cluster_id }} checkpoint delay is more than {threshold} seconds. Please check if the downstream system and network configuration are normal, and also exclude the possibility of an indexed table. If the problem persists, please contact our support team directly for further assistance.
monitorTarget: ' Changefeed processor checkpoint delay '
operator: ' is larger than {threshold} seconds'
ruleID: "10"
severity: major
status: active
visibility: external
- name: data-plane/TiDBCloudIncident
rules:
- alert: IncidentChangefeedHighDelay
annotations:
message: |
The Changefeed {{ $labels.changefeed }} of cluster {{ $labels.cluster_id }} checkpoint delay is more than 8 hours.
expr: ticdc_owner_checkpoint_ts_lag{} > 8*60*60
for: 1m
labels:
component: resilience
severity: major
- alert: IncidentTechnicalSLA
annotations:
message: |
The TiDB cluster {{ $labels.cluster_id }} has technical SLA issue.
expr: "(\n avg_over_time(sum by (cluster_id) (rate(tidb_executor_statement_total[1m]))[5m:])\n
\ < bool on (cluster_id) 0.2 * avg_over_time(\n sum by (cluster_id) (rate(tidb_executor_statement_total[1m]))[6h:]\n
\ )\n) + on (cluster_id) (\n avg_over_time(histogram_quantile(0.99, sum by
(cluster_id, le) (rate(tidb_server_handle_query_duration_seconds_bucket[1m])))[5m:])\n
\ > bool on (cluster_id) 100 * \n histogram_quantile(0.99, sum by
(cluster_id, le) (rate(tidb_server_handle_query_duration_seconds_bucket[6h])))\n)
+ on (cluster_id) (\n (\n avg_over_time(sum by (cluster_id) (increase(tidb_server_execute_error_total{type=~\".*8027|.*8120|.*8021|.*8024|.*9001|.*9002|.*9003|.*9011|.*9012|.*9013\"}[1m]))[5m:])
\n > bool 1\n ) or vector(0)\n) >= 2\n"
labels:
component: resilience
severity: major
- alert: IncidentClusterOfflineStoreProgressPending
annotations:
message: The TiDB Cluster {{ $labels.cluster_id }} offline store progress
pending
expr: |
delta(pd_cluster_progress{action="removing"}[1h]) == 0
for: 1m
labels:
component: resilience
severity: major
- alert: IncidentClusterOnlineStoreProgressPending
annotations:
message: The TiDB Cluster {{ $labels.cluster_id }} online store progress pending
expr: |
delta(pd_cluster_progress{action="preparing"}[1h]) == 0
for: 1m
labels:
component: resilience
severity: major
- name: nodeexporter
rules:
- alert: HostOutOfMemory
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
for: 2m
labels:
severity: warning
annotations:
summary: Host out of memory (instance {{ $labels.instance }})
description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostMemoryUnderMemoryPressure
expr: rate(node_vmstat_pgmajfault[1m]) > 1000
for: 2m
labels:
severity: warning
annotations:
summary: Host memory under memory pressure (instance {{ $labels.instance }})
description: "The node is under heavy memory pressure. High rate of major page faults\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualNetworkThroughputIn
expr: sum by (instance) (rate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100
for: 5m
labels:
severity: warning
annotations:
summary: Host unusual network throughput in (instance {{ $labels.instance }})
description: "Host network interfaces are probably receiving too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualNetworkThroughputOut
expr: sum by (instance) (rate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100
for: 5m
labels:
severity: warning
annotations:
summary: Host unusual network throughput out (instance {{ $labels.instance }})
description: "Host network interfaces are probably sending too much data (> 100 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskReadRate
expr: sum by (instance) (rate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50
for: 5m
labels:
severity: warning
annotations:
summary: Host unusual disk read rate (instance {{ $labels.instance }})
description: "Disk is probably reading too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskWriteRate
expr: sum by (instance) (rate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk write rate (instance {{ $labels.instance }})
description: "Disk is probably writing too much data (> 50 MB/s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Please add ignored mountpoints in node_exporter parameters like
# "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
# Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
- alert: HostOutOfDiskSpace
expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
for: 2m
labels:
severity: warning
annotations:
summary: Host out of disk space (instance {{ $labels.instance }})
description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# Please add ignored mountpoints in node_exporter parameters like
# "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
# Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
- alert: HostDiskWillFillIn24Hours
expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~"tmpfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
for: 2m
labels:
severity: warning
annotations:
summary: Host disk will fill in 24 hours (instance {{ $labels.instance }})
description: "Filesystem is predicted to run out of space within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOutOfInodes
expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
for: 2m
labels:
severity: warning
annotations:
summary: Host out of inodes (instance {{ $labels.instance }})
description: "Disk is almost running out of available inodes (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostInodesWillFillIn24Hours
expr: node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint="/rootfs"} * 100 < 10 and predict_linear(node_filesystem_files_free{mountpoint="/rootfs"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly{mountpoint="/rootfs"} == 0
for: 2m
labels:
severity: warning
annotations:
summary: Host inodes will fill in 24 hours (instance {{ $labels.instance }})
description: "Filesystem is predicted to run out of inodes within the next 24 hours at current write rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskReadLatency
expr: rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk read latency (instance {{ $labels.instance }})
description: "Disk latency is growing (read operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostUnusualDiskWriteLatency
expr: rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0
for: 2m
labels:
severity: warning
annotations:
summary: Host unusual disk write latency (instance {{ $labels.instance }})
description: "Disk latency is growing (write operations > 100ms)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostHighCpuLoad
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
for: 0m
labels:
severity: warning
annotations:
summary: Host high CPU load (instance {{ $labels.instance }})
description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostCpuStealNoisyNeighbor
expr: avg by(instance) (rate(node_cpu_seconds_total{mode="steal"}[5m])) * 100 > 10
for: 0m
labels:
severity: warning
annotations:
summary: Host CPU steal noisy neighbor (instance {{ $labels.instance }})
description: "CPU steal is > 10%. A noisy neighbor is killing VM performances or a spot instance may be out of credit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
# 2000 context switches is an arbitrary number.
# Alert threshold depends on nature of application.
# Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58
- alert: HostContextSwitching
expr: (rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 2000
for: 0m
labels:
severity: warning
annotations:
summary: Host context switching (instance {{ $labels.instance }})
description: "Context switching is growing on node (> 2000 / s)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostSwapIsFillingUp
expr: (1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80
for: 2m
labels:
severity: warning
annotations:
summary: Host swap is filling up (instance {{ $labels.instance }})
description: "Swap is filling up (>80%)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostSystemdServiceCrashed
expr: node_systemd_unit_state{state="failed"} == 1
for: 0m
labels:
severity: warning
annotations:
summary: Host systemd service crashed (instance {{ $labels.instance }})
description: "systemd service crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostPhysicalComponentTooHot
expr: node_hwmon_temp_celsius > 75
for: 5m
labels:
severity: warning
annotations:
summary: Host physical component too hot (instance {{ $labels.instance }})
description: "Physical hardware component too hot\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNodeOvertemperatureAlarm
expr: node_hwmon_temp_crit_alarm_celsius == 1
for: 0m
labels:
severity: critical
annotations:
summary: Host node overtemperature alarm (instance {{ $labels.instance }})
description: "Physical node temperature alarm triggered\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostRaidArrayGotInactive
expr: node_md_state{state="inactive"} > 0
for: 0m
labels:
severity: critical
annotations:
summary: Host RAID array got inactive (instance {{ $labels.instance }})
description: "RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostRaidDiskFailure
expr: node_md_disks{state="failed"} > 0
for: 2m
labels:
severity: warning
annotations:
summary: Host RAID disk failure (instance {{ $labels.instance }})
description: "At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostKernelVersionDeviations
expr: count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1
for: 6h
labels:
severity: warning
annotations:
summary: Host kernel version deviations (instance {{ $labels.instance }})
description: "Different kernel versions are running\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostOomKillDetected
expr: increase(node_vmstat_oom_kill[1m]) > 0
for: 0m
labels:
severity: warning
annotations:
summary: Host OOM kill detected (instance {{ $labels.instance }})
description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostEdacCorrectableErrorsDetected
expr: increase(node_edac_correctable_errors_total[1m]) > 0
for: 0m
labels:
severity: info
annotations:
summary: Host EDAC Correctable Errors detected (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} correctable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostEdacUncorrectableErrorsDetected
expr: node_edac_uncorrectable_errors_total > 0
for: 0m
labels:
severity: warning
annotations:
summary: Host EDAC Uncorrectable Errors detected (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} has had {{ printf \"%.0f\" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkReceiveErrors
expr: rate(node_network_receive_errs_total[2m]) / rate(node_network_receive_packets_total[2m]) > 0.01
for: 2m
labels:
severity: warning
annotations:
summary: Host Network Receive Errors (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} receive errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkTransmitErrors
expr: rate(node_network_transmit_errs_total[2m]) / rate(node_network_transmit_packets_total[2m]) > 0.01
for: 2m
labels:
severity: warning
annotations:
summary: Host Network Transmit Errors (instance {{ $labels.instance }})
description: "Host {{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf \"%.0f\" $value }} transmit errors in the last two minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkInterfaceSaturated
expr: (rate(node_network_receive_bytes_total{device!~"^tap.*"}[1m]) + rate(node_network_transmit_bytes_total{device!~"^tap.*"}[1m])) / node_network_speed_bytes{device!~"^tap.*"} > 0.8 < 10000
for: 1m
labels:
severity: warning
annotations:
summary: Host Network Interface Saturated (instance {{ $labels.instance }})
description: "The network interface \"{{ $labels.device }}\" on \"{{ $labels.instance }}\" is getting overloaded.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostNetworkBondDegraded
expr: (node_bonding_active - node_bonding_slaves) != 0
for: 2m
labels:
severity: warning
annotations:
summary: Host Network Bond Degraded (instance {{ $labels.instance }})
description: "Bond \"{{ $labels.device }}\" degraded on \"{{ $labels.instance }}\".\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostConntrackLimit
expr: node_nf_conntrack_entries / node_nf_conntrack_entries_limit > 0.8
for: 5m
labels:
severity: warning
annotations:
summary: Host conntrack limit (instance {{ $labels.instance }})
description: "The number of conntrack is approaching limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostClockSkew
expr: (node_timex_offset_seconds > 0.05 and deriv(node_timex_offset_seconds[5m]) >= 0) or (node_timex_offset_seconds < -0.05 and deriv(node_timex_offset_seconds[5m]) <= 0)
for: 2m
labels:
severity: warning
annotations:
summary: Host clock skew (instance {{ $labels.instance }})
description: "Clock skew detected. Clock is out of sync. Ensure NTP is configured correctly on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostClockNotSynchronising
expr: min_over_time(node_timex_sync_status[1m]) == 0 and node_timex_maxerror_seconds >= 16
for: 2m
labels:
severity: warning
annotations:
summary: Host clock not synchronising (instance {{ $labels.instance }})
description: "Clock not synchronising. Ensure NTP is configured on this host.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: HostRequiresReboot
expr: node_reboot_required > 0
for: 4h
labels:
severity: info
annotations:
summary: Host requires reboot (instance {{ $labels.instance }})
description: "{{ $labels.instance }} requires a reboot.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"