apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    prometheus: basic
    role: alert-rules
  name: basic-prometheus-rules
  namespace: monitoring
spec:
  groups:
  - name: binlog.rules
    rules:
      - alert: binlog_pump_storage_error_count
        expr: changes(binlog_pump_storage_error_count[1m]) > 0
        labels:
          env: basic
          expr: changes(binlog_pump_storage_error_count[1m]) > 0
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog pump storage write some binlogs failed
          value: '{{ $value }}'
      - alert: binlog_drainer_checkpoint_high_delay
        expr: (time() - binlog_drainer_checkpoint_tso / 1000) > 3600
        for: 1m
        labels:
          env: basic
          expr: (time() - binlog_drainer_checkpoint_tso / 1000) > 3600
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog drainer checkpoint delay more than 1 hour
          value: '{{ $value }}'
      - alert: binlog_pump_write_binlog_rpc_duration_seconds_bucket
        expr: histogram_quantile(0.9, rate(binlog_pump_rpc_duration_seconds_bucket{method="WriteBinlog"}[5m]))
          > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.9, rate(binlog_pump_rpc_duration_seconds_bucket{method="WriteBinlog"}[5m]))
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog pump write binlog RPC latency is too high
          value: '{{ $value }}'
      - alert: binlog_pump_storage_write_binlog_duration_time_bucket
        expr: histogram_quantile(0.9, rate(binlog_pump_storage_write_binlog_duration_time_bucket{type="batch"}[5m]))
          > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.9, rate(binlog_pump_storage_write_binlog_duration_time_bucket{type="batch"}[5m]))
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog pump write binlog to disk is too slow
          value: '{{ $value }}'
      - alert: binlog_pump_storage_available_size_less_than_20G
        expr: binlog_pump_storage_storage_size_bytes{type="available"} < 20 * 1024 * 1024
          * 1024
        for: 10s
        labels:
          env: basic
          expr: binlog_pump_storage_storage_size_bytes{type="available"} < 20 * 1024 *
            1024 * 1024
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog pump storage available size less than 20G
          value: '{{ $value }}'
      - alert: binlog_drainer_execute_duration_time_more_than_10s
        expr: histogram_quantile(0.9, rate(binlog_drainer_execute_duration_time_bucket[1m]))
          > 10
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.9, rate(binlog_drainer_txn_duration_time_bucket[1m]))
            > 10
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog binlog drainer execute_duration_time_more_than_10s
          value: '{{ $value }}'
      - alert: binlog_drainer_checkpoint_tso_no_change_for_1m
        expr: changes(binlog_drainer_checkpoint_tso[1m]) < 1
        labels:
          env: basic
          expr: changes(binlog_drainer_checkpoint_tso[1m]) < 1
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
        {{ $value }}'
          summary: binlog drainer checkpoint tso no change for 1m
          value: '{{ $value }}'
  - name: lightning.rules
    rules:
      - alert: Lightning_import_failure_tables_count
        expr: sum ( lightning_tables{result="failure"} ) > 0
        for: 1m
        labels:
          env: basic
          expr: sum ( lightning_tables{result="failure"} ) > 0
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: Lightning failed to import a table
          value: '{{ $value }}'
  - name: pd.rules
    rules:
      - alert: PD_cluster_offline_tikv_nums
        expr: sum ( pd_cluster_status{type="store_down_count"} ) > 0
        for: 1m
        labels:
          env: basic
          expr: sum ( pd_cluster_status{type="store_down_count"} ) > 0
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_cluster_offline_tikv_nums
          value: '{{ $value }}'
      - alert: PD_etcd_write_disk_latency
        expr: histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m]))
          by (instance,job,le) ) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket[1m]))
            by (instance,job,le) ) > 1
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_etcd_write_disk_latency
          value: '{{ $value }}'
      - alert: PD_miss_peer_region_count
        expr: sum( pd_regions_status{type="miss_peer_region_count"} )  > 100
        for: 1m
        labels:
          env: basic
          expr: sum( pd_regions_status{type="miss_peer_region_count"} )  > 100
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_miss_peer_region_count
          value: '{{ $value }}'
      - alert: PD_cluster_lost_connect_tikv_nums
        expr: (sum ( pd_cluster_status{type="store_disconnected_count"} ) by (instance)
          > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
        for: 1m
        labels:
          env: basic
          expr: (sum ( pd_cluster_status{type="store_disconnected_count"} ) by (instance)
            > 0) and (sum(etcd_server_is_leader) by (instance) > 0)
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_cluster_lost_connect_tikv_nums
          value: '{{ $value }}'
      - alert: PD_cluster_low_space
        expr: (sum(pd_cluster_status{type="store_low_space_count"}) by (instance) > 0)
          and (sum(etcd_server_is_leader) by (instance) > 0)
        for: 1m
        labels:
          env: basic
          expr: (sum(pd_cluster_status{type="store_low_space_count"}) by (instance) >
            0) and (sum(etcd_server_is_leader) by (instance) > 0)
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_cluster_low_space
          value: '{{ $value }}'
      - alert: PD_etcd_network_peer_latency
        expr: histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m]))
          by (To,instance,job,le) ) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(etcd_network_peer_round_trip_time_seconds_bucket[1m]))
            by (To,instance,job,le) ) > 1
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_etcd_network_peer_latency
          value: '{{ $value }}'
      - alert: PD_tidb_handle_requests_duration
        expr: histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type="tso"}[1m]))
          by (instance,job,le) ) > 0.1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{type="tso"}[1m]))
            by (instance,job,le) ) > 0.1
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_tidb_handle_requests_duration
          value: '{{ $value }}'
      - alert: PD_down_peer_region_nums
        expr: sum ( pd_regions_status{type="down_peer_region_count"} ) > 0
        for: 1m
        labels:
          env: basic
          expr: sum ( pd_regions_status{type="down_peer_region_count"} ) > 0
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_down_peer_region_nums
          value: '{{ $value }}'
      - alert: PD_incorrect_namespace_region_count
        expr: sum ( pd_regions_status{type="incorrect_namespace_region_count"} ) > 100
        for: 1m
        labels:
          env: basic
          expr: sum ( pd_regions_status{type="incorrect_namespace_region_count"} ) > 0
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_incorrect_namespace_region_count
          value: '{{ $value }}'
      - alert: PD_pending_peer_region_count
        expr: (sum( pd_regions_status{type="pending_peer_region_count"} ) by (instance)  >
          100) and (sum(etcd_server_is_leader) by (instance) > 0)
        for: 1m
        labels:
          env: basic
          expr: (sum( pd_regions_status{type="pending_peer_region_count"} ) by (instance)  >
            100) and (sum(etcd_server_is_leader) by (instance) > 0)
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_pending_peer_region_count
          value: '{{ $value }}'
      - alert: PD_leader_change
        expr: count( changes(pd_server_tso{type="save"}[10m]) > 0 )   >= 2
        for: 1m
        labels:
          env: basic
          expr: count( changes(pd_server_tso{type="save"}[10m]) > 0 )   >= 2
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: PD_leader_change
          value: '{{ $value }}'
      - alert: TiKV_space_used_more_than_80%
        expr: sum(pd_cluster_status{type="storage_size"}) / sum(pd_cluster_status{type="storage_capacity"})
          * 100  > 80
        for: 1m
        labels:
          env: basic
          expr: sum(pd_cluster_status{type="storage_size"}) / sum(pd_cluster_status{type="storage_capacity"})
            * 100  > 80
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV_space_used_more_than_80%
          value: '{{ $value }}'
      - alert: PD_system_time_slow
        expr: changes(pd_server_tso{type="system_time_slow"}[10m]) >= 1
        for: 1m
        labels:
          env: basic
          expr: changes(pd_server_tso{type="system_time_slow"}[10m]) >= 1
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
          {{ $value }}'
          summary: PD_system_time_slow
          value: '{{ $value }}'
      - alert: PD_no_store_for_making_replica
        expr: increase(pd_checker_event_count{type="replica_checker", name="no_target_store"}[1m])
          > 0
        for: 1m
        labels:
          env: basic
          expr: increase(pd_checker_event_count{type="replica_checker", name="no_target_store"}[1m])
            > 0
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: PD_no_store_for_making_replica
          value: '{{ $value }}'
  - name: tidb.rules
    rules:
      - alert: TiDB_schema_error
        expr: increase(tidb_session_schema_lease_error_total{type="outdated"}[15m]) >
          0
        for: 1m
        labels:
          env: basic
          expr: increase(tidb_session_schema_lease_error_total{type="outdated"}[15m])
            > 0
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB schema error
          value: '{{ $value }}'
      - alert: TiDB_tikvclient_region_err_total
        expr: increase( tidb_tikvclient_region_err_total[10m] )  > 6000
        for: 1m
        labels:
          env: basic
          expr: increase( tidb_tikvclient_region_err_total[10m] )  > 6000
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB tikvclient_backoff_count error
          value: '{{ $value }}'
      - alert: TiDB_domain_load_schema_total
        expr: increase( tidb_domain_load_schema_total{type="failed"}[10m] )  > 10
        for: 1m
        labels:
          env: basic
          expr: increase( tidb_domain_load_schema_total{type="failed"}[10m] )  > 10
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB domain_load_schema_total error
          value: '{{ $value }}'
      - alert: TiDB_monitor_keep_alive
        expr: increase(tidb_monitor_keep_alive_total{job="tidb"}[10m]) < 100
        for: 1m
        labels:
          env: basic
          expr: increase(tidb_monitor_keep_alive_total{job="tidb"}[10m]) < 100
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB monitor_keep_alive error
          value: '{{ $value }}'
      - alert: TiDB_server_panic_total
        expr: increase(tidb_server_panic_total[10m]) > 0
        for: 1m
        labels:
          env: basic
          expr: increase(tidb_server_panic_total[10m]) > 0
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB server panic total
          value: '{{ $value }}'
      - alert: TiDB_memory_abnormal
        expr: go_memstats_heap_inuse_bytes{job="tidb"} > 1e+10
        for: 1m
        labels:
          env: basic
          expr: go_memstats_heap_inuse_bytes{job="tidb"} > 1e+10
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB heap memory usage is over 10 GB
          value: '{{ $value }}'
      - alert: TiDB_query_duration
        expr: histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m]))
          BY (le, instance)) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m]))
            BY (le, instance)) > 1
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB query duration 99th percentile is above 1s
          value: '{{ $value }}'
      - alert: TiDB_server_event_error
        expr: increase(tidb_server_event_total{type=~"server_start|server_hang"}[15m])  >
          0
        for: 1m
        labels:
          env: basic
          expr: increase(tidb_server_event_total{type=~"server_start|server_hang"}[15m])  >
            0
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB server event error
          value: '{{ $value }}'
      - alert: tidb_tikvclient_backoff_total
        expr: increase( tidb_tikvclient_backoff_total[10m] )  > 10
        for: 1m
        labels:
          env: basic
          expr: increase( tidb_tikvclient_backoff_total[10m] )  > 10
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB tikvclient_backoff_count error
          value: '{{ $value }}'
      - alert: TiDB_monitor_time_jump_back_error
        expr: increase(tidb_monitor_time_jump_back_total[10m])  > 0
        for: 1m
        labels:
          env: basic
          expr: increase(tidb_monitor_time_jump_back_total[10m])  > 0
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB monitor time_jump_back error
          value: '{{ $value }}'
      - alert: TiDB_ddl_waiting_jobs
        expr: sum(tidb_ddl_waiting_jobs) > 5
        for: 1m
        labels:
          env: basic
          expr: sum(tidb_ddl_waiting_jobs) > 5
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiDB ddl waiting_jobs too much
          value: '{{ $value }}'
  - name: tikv.rules
    rules:
      - alert: TiKV_memory_used_too_fast
        expr: process_resident_memory_bytes{job=~"tikv",instance=~".*"} - (process_resident_memory_bytes{job=~"tikv",instance=~".*"}
          offset 5m) > 5*1024*1024*1024
        for: 5m
        labels:
          env: basic
          expr: process_resident_memory_bytes{job=~"tikv",instance=~".*"} - (process_resident_memory_bytes{job=~"tikv",instance=~".*"}
            offset 5m) > 5*1024*1024*1024
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
          {{ $value }}'
          summary: TiKV memory used too fast
          value: '{{ $value }}'
      - alert: TiKV_GC_can_not_work
        expr: sum(increase(tikv_gcworker_gc_tasks_vec{task="gc"}[1d])) < 1
        for: 1m
        labels:
          env: basic
          expr: sum(increase(tikv_gcworker_gc_tasks_vec{task="gc"}[1d])) < 1
          level: emergency
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV GC can not work
          value: '{{ $value }}'
      - alert: TiKV_server_report_failure_msg_total
        expr: sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[10m]))
          BY (store_id) > 10
        for: 1m
        labels:
          env: basic
          expr: sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[10m]))
            BY (store_id) > 10
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV server_report_failure_msg_total error
          value: '{{ $value }}'
      - alert: TiKV_channel_full_total
        expr: sum(rate(tikv_channel_full_total[10m])) BY (type, instance) > 0
        for: 1m
        labels:
          env: basic
          expr: sum(rate(tikv_channel_full_total[10m])) BY (type, instance) > 0
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV channel full
          value: '{{ $value }}'
      - alert: TiKV_write_stall
        expr: delta( tikv_engine_write_stall[10m])  > 0
        for: 1m
        labels:
          env: basic
          expr: delta( tikv_engine_write_stall[10m])  > 0
          level: critical
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV write stall
          value: '{{ $value }}'
      - alert: TiKV_raft_log_lag
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by
          (le, instance))  > 5000
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_log_lag_bucket[1m]))
            by (le, instance))  > 5000
          level: critical
        annotations:
          description: 'cluster: basic, instance {{ $labels.instance }}, values:
          {{ $value }}'
          summary: TiKV raftstore log lag more than 5000
          value: '{{ $value }}'
      - alert: TiKV_async_request_snapshot_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="snapshot"}[1m]))
          by (le, instance, type)) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="snapshot"}[1m]))
            by (le, instance, type)) > 1
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV async request snapshot duration seconds more than 1s
          value: '{{ $value }}'
      - alert: TiKV_async_request_write_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="write"}[1m]))
          by (le, instance, type)) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{type="write"}[1m]))
            by (le, instance, type)) > 1
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV async request write duration seconds more than 1s
          value: '{{ $value }}'
      - alert: TiKV_coprocessor_request_wait_seconds
        expr: histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m]))
          by (le, instance, req)) > 10
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.9999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket[1m]))
            by (le, instance, req)) > 10
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV coprocessor request wait seconds more than 10s
          value: '{{ $value }}'
      - alert: TiKV_raftstore_thread_cpu_seconds_total
        expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (instance)  >
          1.6
        for: 1m
        labels:
          env: basic
          expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by
            (instance)  > 1.6
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV raftstore thread CPU seconds is high
          value: '{{ $value }}'
      - alert: TiKV_raft_append_log_duration_secs
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m]))
          by (le, instance)) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m]))
            by (le, instance)) > 1
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV_raft_append_log_duration_secs
          value: '{{ $value }}'
      - alert: TiKV_raft_apply_log_duration_secs
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m]))
          by (le, instance)) > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m]))
            by (le, instance)) > 1
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV_raft_apply_log_duration_secs
          value: '{{ $value }}'
      - alert: TiKV_scheduler_latch_wait_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m]))
          by (le, instance, type))  > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket[1m]))
            by (le, instance, type))  > 1
          level: critical
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV scheduler latch wait duration seconds more than 1s
          value: '{{ $value }}'
      - alert: TiKV_thread_apply_worker_cpu_seconds
        expr: sum(rate(tikv_thread_cpu_seconds_total{name="apply_worker"}[1m])) by (instance)
          > 1.8
        for: 1m
        labels:
          env: basic
          expr: sum(rate(tikv_thread_cpu_seconds_total{name="apply_worker"}[1m])) by (instance)
            > 1.8
          level: critical
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV thread apply worker cpu seconds is high
          value: '{{ $value }}'
      - alert: TiDB_tikvclient_gc_action_fail
        expr: sum(increase(tidb_tikvclient_gc_action_result{type="fail"}[1m])) > 10
        for: 1m
        labels:
          env: basic
          expr: sum(increase(tidb_tikvclient_gc_action_result{type="fail"}[1m])) > 10
          level: critical
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiDB_tikvclient_gc_action_fail
          value: '{{ $value }}'
      - alert: TiKV_leader_drops
        expr: delta(tikv_pd_heartbeat_tick_total{type="leader"}[30s]) < -10
        for: 1m
        labels:
          env: basic
          expr: delta(tikv_pd_heartbeat_tick_total{type="leader"}[30s]) < -10
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV leader drops
          value: '{{ $value }}'
      - alert: TiKV_raft_process_ready_duration_secs
        expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m]))
          by (le, instance, type)) > 2
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='ready'}[1m]))
            by (le, instance, type)) > 2
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
          {{ $value }}'
          summary: TiKV_raft_process_ready_duration_secs
          value: '{{ $value }}'
      - alert: TiKV_raft_process_tick_duration_secs
        expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m]))
          by (le, instance, type)) > 2
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.999, sum(rate(tikv_raftstore_raft_process_duration_secs_bucket{type='tick'}[1m]))
            by (le, instance, type)) > 2
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
          {{ $value }}'
          summary: TiKV_raft_process_tick_duration_secs
          value: '{{ $value }}'
      - alert: TiKV_scheduler_context_total
        expr: abs(delta( tikv_scheduler_contex_total[5m])) > 1000
        for: 1m
        labels:
          env: basic
          expr: abs(delta( tikv_scheduler_contex_total[5m])) > 1000
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV scheduler context total
          value: '{{ $value }}'
      - alert: TiKV_scheduler_command_duration_seconds
        expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m]))
          by (le, instance, type)  / 1000)  > 1
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket[1m]))
            by (le, instance, type)  / 1000)  > 1
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:{{
          $value }}'
          summary: TiKV scheduler command duration seconds more than 1s
          value: '{{ $value }}'
      - alert: TiKV_coprocessor_outdated_request_wait_seconds
        expr: delta( tikv_coprocessor_outdated_request_wait_seconds_count[10m] )  > 0
        for: 1m
        labels:
          env: basic
          expr: delta( tikv_coprocessor_outdated_request_wait_seconds_count[10m] )  >
            0
          level: warning
        annotations:
          description: 'cluster: basic, instance: {{ $labels.instance }}, values:
          {{ $value }}'
          summary: TiKV coprocessor outdated request wait seconds
          value: '{{ $value }}'
      - alert: TiKV_coprocessor_request_error
        expr: increase(tikv_coprocessor_request_error{reason!="lock"}[10m]) > 100
        for: 1m
        labels:
          env: basic
          expr: increase(tikv_coprocessor_request_error{reason!="lock"}[10m]) > 100
          level: warning
        annotations:
          description: 'cluster: basic, reason: {{ $labels.reason }}, instance:
          {{ $labels.instance }}, values: {{ $value }}'
          summary: TiKV coprocessor request error
          value: '{{ $value }}'
      - alert: TiKV_coprocessor_request_lock_error
        expr: increase(tikv_coprocessor_request_error{reason="lock"}[10m]) > 10000
        for: 1m
        labels:
          env: basic
          expr: increase(tikv_coprocessor_request_error{reason="lock"}[10m]) > 10000
          level: warning
        annotations:
          description: 'cluster: basic, reason: {{ $labels.reason }}, instance:
          {{ $labels.instance }}, values: {{ $value }}'
          summary: TiKV coprocessor request lock error
          value: '{{ $value }}'
      - alert: TiKV_coprocessor_pending_request
        expr: delta( tikv_coprocessor_pending_request[10m]) > 5000
        for: 1m
        labels:
          env: basic
          expr: delta( tikv_coprocessor_pending_request[10m]) > 5000
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV pending {{ $labels.type }} request is high
          value: '{{ $value }}'
      - alert: TiKV_batch_request_snapshot_nums
        expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"cop_.*"}[1m])) by (instance)
          / ( count(tikv_thread_cpu_seconds_total{name=~"cop_.*"}) *  0.9 ) / count(count(tikv_thread_cpu_seconds_total)
          by (instance)) > 0
        for: 1m
        labels:
          env: basic
          expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"cop_.*"}[1m])) by (instance)
            / ( count(tikv_thread_cpu_seconds_total{name=~"cop_.*"}) *  0.9 ) / count(count(tikv_thread_cpu_seconds_total)
            by (instance)) > 0
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV batch request snapshot nums is high
          value: '{{ $value }}'
      - alert: TiKV_pending_task
        expr: sum(tikv_worker_pending_task_total) BY (instance,name)  > 1000
        for: 1m
        labels:
          env: basic
          expr: sum(tikv_worker_pending_task_total) BY (instance,name)  > 1000
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV pending task too much
          value: '{{ $value }}'
      - alert: TiKV_low_space_and_add_region
        expr: count( (sum(tikv_store_size_bytes{type="available"}) by (instance) / sum(tikv_store_size_bytes{type="capacity"})
          by (instance) < 0.2) and (sum(tikv_raftstore_snapshot_traffic_total{type="applying"})
          by (instance) > 0 ) ) > 0
        for: 1m
        labels:
          env: basic
          expr: count( (sum(tikv_store_size_bytes{type="available"}) by (instance) / sum(tikv_store_size_bytes{type="capacity"})
            by (instance) < 0.2) and (sum(tikv_raftstore_snapshot_traffic_total{type="applying"})
            by (instance) > 0 ) ) > 0
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV low_space and add_region
          value: '{{ $value }}'
      - alert: TiKV_approximate_region_size
        expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket[1m]))
          by (le)) > 1073741824
        for: 1m
        labels:
          env: basic
          expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_region_size_bucket[1m]))
            by (le)) > 1073741824
          level: warning
        annotations:
          description: 'cluster: basic, type: {{ $labels.type }}, instance: {{
          $labels.instance }}, values: {{ $value }}'
          summary: TiKV approximate region size is more than 1GB
          value: '{{ $value }}'