groups:
- name: pd/pd
  rules:
  - alert: PDServerIsDown
    expr: up{component="pd"} == 0
    for: 10m
    labels:
      severity: critical
      response_class: page
      expr: up{component="pd"} == 0
      tier: serverless
      component: pd
      oncall_team: serverless
      provider_type: aws-free-tier
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.f5wwrkav7k5f
      value: '{{ $value }}'
      summary: PD server is down
  - alert: PDMcsNoLeaderOrPrimary
    expr: count_over_time((sum(service_member_role) by (job, service) != 1)[5m:1m]) > 2
    labels:
      severity: critical
      response_class: page
      expr: count_over_time((sum(service_member_role) by (job, service) != 1)[5m:1m]) > 2
      tier: serverless
      component: pd
      oncall_team: serverless
      provider_type: aws-free-tier
    annotations:
      message: 'job: {{ $labels.job }}, service: {{ $labels.service }}, abnormal samples in 5m: {{ $value }}'
      runbook_url:
      value: '{{ $value }}'
      summary: PD service member role is abnormal
  - alert: PDDiscoverOfflineStore
    expr: (sum ( pd_cluster_status{type="store_down_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr:  (sum ( pd_cluster_status{type="store_down_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.zigyfcz6fd17
      value: '{{ $value }}'
      summary: "PD found store is down"
  - alert: PDDiscoverLowSpaceStore
    expr: (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr:  (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(etcd_server_is_leader) by (cluster_id, instance, tenant, provider_type) > 0)
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t3cn3udx76m7
      value: '{{ $value }}'
      summary: "PD found some store space is too low"
  - alert: PDRegionDownPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="down-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.2boq1zqje9s
      value: '{{ $value }}'
      summary: "PD found some region has down peer too long duration"
  - alert: PDRegionMissPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: histogram_quantile(0.99, sum(rate(pd_regions_abnormal_peer_duration_seconds_bucket{type="miss-voter-peer"}[5m])) by (le, type, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.my2gfezc47ow
      value: '{{ $value }}'
      summary: "PD found some region has miss votor peer too long duration"
  - alert: TiKVRegionPendingPeerTooLong
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m])) BY (le, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_peer_pending_duration_seconds_bucket[1m])) BY (le, instance, cluster_id, tenant, provider_type)) > (60 * 60 * 2)
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.lizm9t5f8fpu
      value: '{{ $value }}'
      summary: "TiKV found some region has pending peer too long duration"
  - alert: GCSafePointNotAdvancing
    expr: (max(pd_gc_gc_safepoint) - max((pd_gc_gc_safepoint offset 24h))) == 0
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: (max(pd_gc_gc_safepoint) - max((pd_gc_gc_safepoint offset 24h))) == 0
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "GC safepoint has not advanced for more than 3 hours"
  - alert: GCSafePointAbsent
    expr:  absent(pd_gc_gc_safepoint) > 0
    for: 30m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: absent(pd_gc_gc_safepoint) > 0
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "GC safepoint metrics absent for 30 minutes"
  - alert: OperatorTimeout
    expr: sum(delta(pd_schedule_operators_count{event="timeout"}[1m])) > 0
    for: 10m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(delta(pd_schedule_operators_count{event="timeout"}[1m])) > 0
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD operator timeout"
  - alert: RUTokenUseTooFast
    expr: (max(label_replace(resource_manager_resource_unit_request_unit_consume_rate, "keyspace_id", "$1", "resource_group", "(.*)")) by (keyspace_id) - on (keyspace_id) group_right() sum by (keyspace_id, cluster_id, kubernetes_namespace) (0*tidb_server_connections)) > 0.8
    for: 5m
    labels:
      severity: warning
      tier: serverless
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, namespace: {{ $labels.kubernetes_namespace }}, keyspace_id: {{ $labels.keyspace_id }}, value: {{ $value }}'
      summary: "RU Token use too fast"
  - alert: TSONotLeader
    expr: sum(increase(pd_tso_events{type="not_leader"}[1m])) by (group, dc, type) > 2000
    for: 2m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(increase(pd_tso_events{type="not_leader"}[1m])) by (group, dc, type) > 2000
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD TSO Not Leader"
  - alert: PDMcsGoroutinesTooHigh
    expr: go_goroutines{job=~".*tso.*|.*resource-manager.*|.*sche.*"} > 10000
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'job: {{ $labels.job }}, instance: {{ $labels.instance }}, pod: {{ $labels.pod }}, goroutines: {{ $value }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD microservice goroutines exceed 10000"
  - alert: PDClusterHighCPU
    expr: (sum(rate(container_cpu_usage_seconds_total{namespace="tidb-serverless", pod=~"serverless-cluster-.*pd-.*", container!="POD", container!=""}[2m])) by (pod) / sum(kube_pod_container_resource_limits{namespace="tidb-serverless", pod=~"serverless-cluster-.*pd-.*", resource="cpu"}) by (pod)) > 0.7
    for: 2m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD CPU usage exceeds the threshold"
  - alert: PDClusterCPUThrottled
    expr: (sum(rate(container_cpu_cfs_throttled_periods_total{namespace="tidb-serverless", pod=~"serverless-cluster-.*pd-.*", container!="POD", container!=""}[2m])) by (pod) / sum(rate(container_cpu_cfs_periods_total{namespace="tidb-serverless", pod=~"serverless-cluster-.*pd-.*", container!="POD", container!=""}[2m])) by (pod)) > 0.3
    for: 2m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, pod: {{ $labels.pod }}, CPU throttle ratio is {{ $value | humanizePercentage }}, exceeds 30% threshold.'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD CPU throttling exceeds the threshold"
  - alert: PDClusterHighMemory
    expr: (sum(container_memory_working_set_bytes{namespace="tidb-serverless", pod=~"serverless-cluster-.*pd-.*", container!="POD", container!="", image!=""}) by (namespace, pod) / sum(kube_pod_container_resource_limits{namespace="tidb-serverless", pod=~"serverless-cluster-.*pd-.*", resource="memory"}) by (namespace, pod)) > 0.7
    for: 2m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, pod: {{ $labels.pod }}, PD memory usage is {{ $value | humanizePercentage }}, exceeds 70% threshold.'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD memory usage exceeds the threshold"
  - alert: PDClusterHighDisk
    expr: |
      (
        kubelet_volume_stats_used_bytes{namespace="tidb-serverless", persistentvolumeclaim=~"pd-serverless-cluster-(pd|standby-pd|witness-pd)-.*"} /
        kubelet_volume_stats_capacity_bytes{namespace="tidb-serverless", persistentvolumeclaim=~"pd-serverless-cluster-(pd|standby-pd|witness-pd)-.*"}
      )
      * on (namespace, persistentvolumeclaim) group_left(pod)
      kube_pod_spec_volumes_persistentvolumeclaims_info{namespace="tidb-serverless", pod=~"serverless-cluster-(pd|standby-pd|witness-pd)-.*"}
      > 0.7
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'PD pod {{ $labels.pod }} pvc {{ $labels.persistentvolumeclaim }} disk usage exceeds 70% ({{ $value }}).'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD disk usage exceeds the threshold"
  - alert: PDEtcdMVCCDBTotalSizeTooLarge
    expr: etcd_mvcc_db_total_size_in_bytes{component="pd"} > 6 * 1024 * 1024 * 1024
    for: 5m
    labels:
      severity: warning
      tier: serverless
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, pod: {{ $labels.pod }}, instance: {{ $labels.instance }}, etcd MVCC DB total size is {{ $value }} bytes, exceeds 6 GiB.'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD etcd MVCC DB total size exceeds 6 GiB"
  - alert: PDEtcdMVCCDBTotalSizeInUseTooLarge
    expr: etcd_mvcc_db_total_size_in_use_in_bytes{component="pd"} > 4 * 1024 * 1024 * 1024
    for: 5m
    labels:
      severity: warning
      tier: serverless
      component: pd
      oncall_team: serverless
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, pod: {{ $labels.pod }}, instance: {{ $labels.instance }}, etcd MVCC DB total size in use is {{ $value }} bytes, exceeds 4 GiB.'
      runbook_url:
      value: '{{ $value }}'
      summary: "PD etcd MVCC DB total size in use exceeds 4 GiB"