groups:
- name: tidb-cluster
  rules:
  - alert: TiDBServerIsDown
    expr: up{component="tidb", job=~".*tidb", tenant=~"1372813089208611456|1372813089195441286|1372813089194211283|1372813089209212842|1372813089190491281|1372813089209061633|1372813089209001464"} == 0
    for: 5m
    labels:
      severity: warning
      component: tidb
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }} is down for 5 mins'
      description: One TiDB server is down
  - alert: TiKVServerIsDown
    expr: up{component="tikv", job=~".*tikv", tenant=~"1372813089208611456|1372813089195441286|1372813089194211283|1372813089209212842|1372813089190491281|1372813089209061633|1372813089209001464"} == 0
    for: 5m
    labels:
      severity: warning
      component: tikv
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }} is down for 5 mins'
      description: One TiKV server is down
  - alert: TiFlashServerIsDown
    expr: up{component="tiflash",job=~".*tiflash", tenant=~"1372813089208611456|1372813089195441286|1372813089209061633|1372813089194211283|1372813089209212842|1372813089190491281|1372813089209001464"} == 0
    for: 5m
    labels:
      severity: warning
      component: tiflash
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }} is down for 5 mins'
      description: One TiFlash server is down
  - alert: PDServerIsDown
    expr: up{component="pd", job=~".*pd", tenant=~"1372813089208611456|1372813089195441286|1372813089194211283|1372813089209212842|1372813089190491281|1372813089209001464"} == 0
    for: 5m
    labels:
      severity: warning
      component: pd
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }} is down for 5 mins '
      description: PD server is down
  - alert: TiProxyServerIsDown
    expr: up{component="tiproxy", tenant=~"1372813089208611456|1372813089195441286|1372813089194211283"} == 0
    for: 5m
    labels:
      severity: warning
      component: tiproxy
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }} is down for 5 mins'
      description: One TiProxy server is down
  # TiDBAvgRequestLatencyTooLong Alert for BingoPlus
  - alert: TiDBAvgKVRequestLatencyTooLong
    expr: sum(rate(tidb_tikvclient_request_seconds_sum{tenant="1372813089209238420", store!="0"}[2m])) by (type, tidb_cluster_id)/ sum(rate(tidb_tikvclient_request_seconds_count{tenant="1372813089209238420", store!="0"}[2m])) by (type, tidb_cluster_id) > 1
    for: 5m
    labels:
      severity: warning
      component: tidb
      visibility: external
    annotations:
      message: 'TiDB average KV request latency more than 1 second, cluster: {{ $labels.tidb_cluster_id }}, request type: {{ $labels.type }}'
      description: 'TiDB average KV request latency more than 1 second'
  - alert: TiDBServerRestarted
    expr: |-
      ((time() - process_start_time_seconds{component="tidb", tenant=~"1372813089209061633|1372813089209238420"}) < 120) and (process_start_time_seconds offset 5m)
    for: 0s
    labels:
      severity: warning
      component: tidb
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }} is down for at least 1 times in an hour'
      description: 'TiDB server restarts at least 1 times in an hour'
  - alert: TiKVServerRestarted
    expr: |-
      ((time() - process_start_time_seconds{component="tikv", tenant=~"1372813089209061633|1372813089209238420"}) < 120) and (process_start_time_seconds offset 5m)
    for: 0s
    labels:
      severity: warning
      component: tikv
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }} is down for at least 1 times in an hour'
      description: 'TiKV server restarts at least 1 times in an hour'
  - alert: TiFlashServerRestarted
    expr: |-
      ((time() - tiflash_proxy_process_start_time_seconds{component="tiflash", tenant=~"1372813089209061633|1372813089209238420"}) < 120) and (tiflash_proxy_process_start_time_seconds offset 5m)
    for: 0s
    labels:
      severity: warning
      component: tiflash
      visibility: external
    annotations:
      message: 'cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }} is down for at least 1 times in an hour'
      description: 'TiFlash server restarts at least 1 times in an hour'