groups:
- name: alert.rules
  rules:
  - alert: TiDB_schema_error
    expr: increase(tidb_session_schema_lease_error_total{type="outdated"}[15m]) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: emergency
      expr:  increase(tidb_session_schema_lease_error_total{type="outdated"}[15m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB schema error

  - alert: TiDB_tikvclient_region_err_total
    expr: increase( tidb_tikvclient_region_err_total[10m] )  > 6000
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: emergency
      expr:  increase( tidb_tikvclient_region_err_total[10m] )  > 6000
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB tikvclient_backoff_count error

  - alert: TiDB_binlog_error_total
    expr: increase( tidb_server_critical_error_total[5m] )  > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: emergency
      expr:  increase( tidb_server_critical_error_total[5m] )  > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB tidb binlog error total

  - alert: TiDB_domain_load_schema_total
    expr: increase( tidb_domain_load_schema_total{type="failed"}[10m] )  > 10
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: emergency
      expr:  increase( tidb_domain_load_schema_total{type="failed"}[10m] )  > 10
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB domain_load_schema_total error

  - alert: TiDB_monitor_keep_alive
    expr: increase(tidb_monitor_keep_alive_total{job="tidb"}[10m]) < 100
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: emergency
      expr: increase(tidb_monitor_keep_alive_total{job="tidb"}[10m]) < 100
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB monitor_keep_alive error

  - alert: TiDB_server_panic_total
    expr: increase(tidb_server_panic_total[10m]) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr:  increase(tidb_server_panic_total[10m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB server panic total

  - alert: TiDB_memory_abnormal
    expr: go_memstats_heap_inuse_bytes{job="tidb"} > 1e+10
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: go_memstats_heap_inuse_bytes{job="tidb"} > 1e+10
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB heap memory usage is over 10 GB

  - alert: TiDB_query_duration
    expr: histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) BY (le, instance)) > 1
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr:  histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) BY (le, instance)) > 1
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB query duration 99th percentile is above 1s

  - alert: TiDB_server_event_error
    expr: increase(tidb_server_event_total{type=~"server_start|server_hang"}[15m])  > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr:  increase(tidb_server_event_total{type=~"server_start|server_hang"}[15m])  > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB server event error

  - alert: tidb_tikvclient_backoff_seconds_count
    expr: increase( tidb_tikvclient_backoff_seconds_count[10m] )  > 10
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr:  increase( tidb_tikvclient_backoff_seconds_count[10m] )  > 10
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB tikvclient_backoff_count error

  - alert: TiDB_monitor_time_jump_back_error
    expr: increase(tidb_monitor_time_jump_back_total[10m])  > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr:  increase(tidb_monitor_time_jump_back_total[10m])  > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB monitor time_jump_back error

  - alert: TiDB_ddl_waiting_jobs
    expr: sum(tidb_ddl_waiting_jobs) > 5
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr:  sum(tidb_ddl_waiting_jobs) > 5
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB ddl waiting_jobs too much

  - alert: TiDB_node_restart
    expr: changes(process_start_time_seconds{job="tidb"}[5m]) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr:  changes(process_start_time_seconds{job="tidb"}[5m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiDB server has been restarted