groups:
- name: pd
  rules:
  - alert: PDServerFrequentlyRestart
    expr: |-
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*pd.*"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*pd.*"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      oncall_team: pd
    annotations:
      description: 'PD server {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has restarted {{ $value }} times in the past hour. Frequent restarts may indicate underlying issues with the PD server or infrastructure. Please investigate the root cause and ensure PD server stability. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.u233t02connq
      summary: PD server restarted {{ $value }} times in past an hour
  - alert: PDClusterFrequentlyRestart
    expr: |-
      (
        sum by (tidb_cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*pd.*"}))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*pd.*"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*pd.*"}))[1h:15s])
            ) by (tidb_cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (tidb_cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      oncall_team: pd
    annotations:
      description: 'Multiple PD servers in cluster {{ $labels.tidb_cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting PD stability. Please investigate the cluster infrastructure and PD configuration to prevent further restarts. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.u233t02connq
      summary: PD servers in a cluster restarted {{ $value }} times in past an hour
  - alert: PDRegionDownPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m])) by (le, type, instance, tidb_cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: next-gen
      expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m])) by (le, type, instance, tidb_cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: pd
    annotations:
      description: 'PD in cluster {{ $labels.tidb_cluster_id }} has detected regions with down peers that have been unavailable for more than 2 hours. This may affect data availability and cluster performance. Please check the TiKV nodes and network connectivity to resolve the down peer issue. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.2boq1zqje9s
      value: '{{ $value }}'
      summary: "PD found some region has down peer too long duration"
  - alert: PDRegionMissPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m])) by (le, type, instance, tidb_cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: next-gen
      expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m])) by (le, type, instance, tidb_cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: pd
    annotations:
      description: 'PD in cluster {{ $labels.tidb_cluster_id }} has detected regions with missing voter peers for more than 2 hours. This is a critical issue that affects data consistency and availability. Please immediately check TiKV nodes and resolve the missing voter peer issue to maintain cluster integrity. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.my2gfezc47ow
      value: '{{ $value }}'
      summary: "PD found some region has miss votor peer too long duration"
  - alert: TiKVRegionPendingPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m])) BY (le, instance, tidb_cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: next-gen
      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m])) BY (le, instance, tidb_cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: pd
    annotations:
      description: 'TiKV in cluster {{ $labels.tidb_cluster_id }} has detected regions with pending peers that have been in pending state for more than 2 hours. This may indicate slow region scheduling or TiKV performance issues. Please check TiKV performance and PD scheduling configuration. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.lizm9t5f8fpu
      value: '{{ $value }}'
      summary: "TiKV found some region has pending peer too long duration"