groups:
- name: pd
  rules:
  - alert: PDServerDowntimeIsTooLong
    expr: up{component="pd", job=~".*pd"} == 0
    for: 10m
    labels:
      severity: critical
      response_class: page
      expr: up{component="pd", job=~".*pd"} == 0
      tier: dedicated
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.f5wwrkav7k5f
      value: '{{ $value }}'
      summary: PD server downtime is more than 10 min
  - alert: PDServerFrequentlyRestart
    expr: sum(changes(floor(process_start_time_seconds{job=~".*pd"})[1h:15s])) by (cluster_id, instance, tenant,provider_type) >= 3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      expr: sum(changes(floor(process_start_time_seconds{job=~".*pd"})[1h:15s])) by (cluster_id, instance, tenant,provider_type) >= 3
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.u233t02connq
      summary: PD server restarted {{ $value }} times in past an hour
  - alert: PDClusterFrequentlyRestart
    expr: |-
      sum by (cluster_id, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*pd"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*pd"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.u233t02connq
      summary: PD servers in a cluster restarted {{ $value }} times in past an hour
  - alert: PDDiscoverDownStore
    expr: (sum ( pd_cluster_status{type="store_down_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: dedicated
      expr:  (sum ( pd_cluster_status{type="store_down_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.zigyfcz6fd17
      value: '{{ $value }}'
      summary: "PD found store is down"
  - alert: PDDiscoverLowSpaceStore
    expr: (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
    for: 5m
    labels:
      severity: warning
      tier: dedicated
      expr:  (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t3cn3udx76m7
      value: '{{ $value }}'
      summary: "PD found some store space is too low"
  - alert: PDRegionDownPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: warning
      tier: dedicated
      expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.2boq1zqje9s
      value: '{{ $value }}'
      summary: "PD found some region has down peer too long duration"
  - alert: PDRegionMissPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: dedicated
      expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.my2gfezc47ow
      value: '{{ $value }}'
      summary: "PD found some region has miss votor peer too long duration"
  # mute on 9.19, 2025. confirmed by Lei Gao.
  # - alert: TiKVRegionPendingPeerTooLong
  #   expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m])) BY (le, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
  #   for: 5m
  #   labels:
  #     severity: major
  #     tier: dedicated
  #     expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m])) BY (le, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
  #     component: pd
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.lizm9t5f8fpu
  #     value: '{{ $value }}'
  #     summary: "TiKV found some region has pending peer too long duration"
  - alert: EtcdNodeTermLagTooLong
    expr: (pd_server_etcd_state{type="term"} or on() vector(0)) < max(pd_server_etcd_state{type="term"})
    for: 2m
    labels:
      severity: warning
      expr: (pd_server_etcd_state{type="term"} or on() vector(0)) < max(pd_server_etcd_state{type="term"})
      tier: dedicated
      component: pd
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      summary: some PD instance term is lagged too long

# PDLeaderChange & PDTSOFailed move to rules/dedicated/swat/data-plane/kernel-tcocp.yaml