groups:
- name: next-gen-pd-kernel-tcocp.rules
  rules:
  # Disable the following rule for ci check
  # ignore_validations: expressionDoesNotUseRangeShorterThan
  - alert: PDLeaderChangeNextGen
    expr: |-
      changes(service_member_role{service="PD"}[15s]) > 0
      unless on(instance)
      (
        (
          sum(
            count_over_time(
              pd_service_maxprocs{component=~".*pd.*"}[30s]
            )
          ) by (instance)
          or on(instance)
          0 * pd_service_maxprocs{component=~".*pd.*"}
        ) <= 1
      )
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: pd
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: pd
      oncall_team: pd
      source_tcoc: TCOC-3517
    annotations:
      summary: pd leader changes, might cause perfermance degrade.
      description: 'PD leader changes, might cause perfermance degrade. source_tcoc: TCOC-3517. PD leader has changed in cluster {{ $labels.cluster_id }}. The new leader is {{ $labels.instance }}. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  - alert: PDTSOFailedNextGen
    expr: increase(pd_tso_events{type="exceeded_max_retry"}[10m]) > 0
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: pd
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: pd
      oncall_team: pd
      source_tcoc: TCOC-3599
    annotations:
      summary: pd tso failed
      description: 'pd tso failed source_tcoc: TCOC-3599 cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  - alert: PDOOMRelatedPodRestartsNextGen
    expr: |-
      sum by(namespace, pod) (
        increase(kube_pod_container_status_restarts_total{container="pd",pod=~"db-[a-z0-9]+-pd-.+"}[15m]) > 0
      )
      * on (namespace, pod) group_left ()
        max by(namespace, pod) (
          kube_pod_container_status_last_terminated_reason{container="pd",pod=~"db-[a-z0-9]+-pd-.+",reason="OOMKilled"} > 0
        )
    for: 0s
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: pd
      oncall_team: pd
      source_tcoc: TCOC-4157
    annotations:
      message: |
        PD Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled. source_tcoc: TCOC-4157