groups:
- name: TiDBCloudIncident
  rules:
  - alert: DiskReadLatencyTooHigh
    annotations:
      message: The disk read latency for {{ $labels.pod }} in cluster {{ $labels.namespace }} is over 5ms.
    expr: |
      label_replace(
        (
          sum by (tenant, instance, device) (rate(node_disk_read_time_seconds_total{tenant=~"1372813089209061633|1372813089209238420|1372813089209256793", device!~"dm-0|nvme0n1|nvme1n1"}[2m]))
          / sum by (tenant, instance, device) (rate(node_disk_reads_completed_total{tenant=~"1372813089209061633|1372813089209238420|1372813089209256793", device!~"dm-0|nvme0n1|nvme1n1"}[2m]))
        ) * on (tenant, instance) group_left(namespace, pod)
        (label_replace(kube_pod_info{tenant=~"1372813089209061633|1372813089209238420|1372813089209256793", namespace=~"tidb.*", namespace!="tidb-admin", pod=~"db-tikv.*|db-tiflash.*"}, "instance", "$0", "node", ".*"))
      , "cluster_id", "$1", "namespace", "tidb(.*)") > 0.005
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: DiskWriteLatencyTooHigh
    annotations:
      message: The disk write latency for {{ $labels.pod }} in cluster {{ $labels.namespace }} is over 5ms.
    expr: |
      label_replace(
        (
          sum by (tenant, instance, device) (rate(node_disk_write_time_seconds_total{tenant=~"1372813089209061633|1372813089209238420|1372813089209256793", device!~"dm-0|nvme0n1|nvme1n1"}[2m]))
          / sum by (tenant, instance,device) (rate(node_disk_writes_completed_total{tenant=~"1372813089209061633|1372813089209238420|1372813089209256793", device!~"dm-0|nvme0n1|nvme1n1"}[2m]))
        ) * on (tenant, instance) group_left(namespace, pod)
        (label_replace(kube_pod_info{tenant=~"1372813089209061633|1372813089209238420|1372813089209256793", namespace=~"tidb.*", namespace!="tidb-admin", pod=~"db-tikv.*|db-tiflash.*"}, "instance", "$0", "node", ".*"))
      , "cluster_id", "$1", "namespace", "tidb(.*)") > 0.005
    for: 60s
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience