groups:
  - name: csng.grpc
    rules:
      - alert: gRPCServerErrorRate > 10%
        expr: |
          (
            sum by (grpc_service, grpc_method, job) (
              rate(grpc_server_handled_total{grpc_code!="OK",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="NotFound",grpc_method="GetTiDBGroup",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="DeleteTiDBCluster",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="NotFound",grpc_method="GetPrivateLinkService",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="NotFound",grpc_method="GetSharedPool",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="NotFound",grpc_method="GetPlacementPolicy",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="NotFound",grpc_method="GetCassandraProxy",namespace=~"cluster-ng-service"}[5m])
              unless
              rate(grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="UpdateTiDBGroup",namespace=~"cluster-ng-service"}[5m])
            )
            /
            sum by (grpc_service, grpc_method, job) (
              rate(
                grpc_server_handled_total{namespace=~"cluster-ng-service"}[5m]
              )
            )
          ) * 100
          > 10
          and
          sum by (grpc_service, grpc_method, job) (
            increase(
              grpc_server_handled_total{namespace=~"cluster-ng-service"}[5m]
            )
          )
          > 10
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen
        annotations:
          message: gRPC server on region {{ $labels.tidb_cluster_id }} failure rate is {{ printf "%.2f" $value }}%, method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}, component={{ $labels.job }}.