groups:
  - name: dataflow-service
    rules:
      - alert: DataflowPodRestartsFrequently
        annotations:
          summary: The dataflow-service pod restart frequently
          message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times in 30 minutes.
        expr: |
          increase(kube_pod_container_status_restarts_total{container=~"dataflow.*"}[30m]) > 1
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: DataflowGrpcErrorQPSTooHigh
        annotations:
          summary: The dataflow-service grpc error qps too high
          message: GRPC Service {{ $labels.grpc_method }}, GRPC Method{{ $labels.grpc_method }} error qps too high in 5 minutes.
        expr: |
          sum(rate(grpc_server_handled_total{namespace=~"prod-ms", pod=~"dataflow(.*)", grpc_service=~"dataflow(.*)", grpc_type="unary", grpc_code!="OK"}[5m])) by (grpc_service, grpc_method, grpc_code) > 0.1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools