groups:
  - name: dp-infra-provider
    rules:
      - alert: DataMigrationProviderOOMKilled
        expr: increase(kube_pod_container_status_restarts_total{pod=~"dp-((eks)|(gke))-infra-provider-[^-]+-[^-]+$"}[5m]) >= 2 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled", pod=~"dp-((eks)|(gke))-infra-provider-[^-]+-[^-]+$"} == 1
        for: 3m
        labels:
          component: data-migration
          oncall_team: tidb_x_tools
          severity: critical
          response_class: page
        annotations:
          description: "EKS/GKE Provider pod {{ $labels.pod }} in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      - alert: TicdcOrBrProviderOOMKilled
        expr: increase(kube_pod_container_status_restarts_total{pod=~"dp-((eks)|(gke))-infra-provider-((br)|(cdc))-.*"}[5m]) >= 2 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{reason="OOMKilled", pod=~"dp-((eks)|(gke))-infra-provider-((br)|(cdc))-.*"} == 1
        for: 3m
        labels:
          component: dataflow-service
          oncall_team: tidb_x_tools
          severity: critical
          response_class: page
        annotations:
          description: "EKS/GKE Provider pod {{ $labels.pod }} in {{ $labels.cluster }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      - alert: InfraProviderPanic
        expr: increase(infra_provider_controller_panic{controller=~"br|cdc"}[1h])> 5
        for: 1m
        labels:
          component: dataflow-service
          oncall_team: tidb_x_tools
          severity: critical
          response_class: page
        annotations:
          description: The Controller {{ $labels.controller }} raises {{ $value }} panics in 1 hour(s).
      - alert: InfraProviderPanic
        expr: increase(infra_provider_controller_panic{controller=~"dwworker|dumpling"}[1h])> 5
        for: 1m
        labels:
          component: data-integration
          oncall_team: tidb_x_tools
          severity: critical
          response_class: page
        annotations:
          description: The Controller {{ $labels.controller }} raises {{ $value }} panics in 1 hour(s).
      - alert: InfraProviderPanic
        expr: increase(infra_provider_controller_panic{controller=~"import"}[1h])> 5
        for: 1m
        labels:
          component: lightning
          oncall_team: tidb_x_tools
          severity: critical
          response_class: page
        annotations:
          description: The Controller {{ $labels.controller }} raises {{ $value }} panics in 1 hour(s).