groups:
  - name: mgmt.grpc
    rules:
      - alert: gRPCServerErrorRateCritical
        expr: |
          100 *
          (
            sum by (grpc_service, grpc_method) (
              rate(
                grpc_server_handled_total{
                  namespace="tidb-management-service",
                  job="tidb-mgmt-service",
                  grpc_code=~"Code.5.+"
                }[5m]
              )
            )
            /
            clamp_min(
              sum by (grpc_service, grpc_method) (
                rate(
                  grpc_server_handled_total{
                    namespace="tidb-management-service",
                    job="tidb-mgmt-service"
                  }[5m]
                )
              ),
              1
            )
          ) > 5
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
        annotations:
          message: Internal error (bizcode 5xxxxxxx) rate {{ printf "%.2f" $value }}% grpc_service={{ $labels.grpc_service }} grpc_method={{ $labels.grpc_method }}
#
#      - alert: gRPCServerP95LatencyHigh
#        expr: |
#          histogram_quantile(
#            0.95,
#            sum by (grpc_service, grpc_method, le) (
#              rate(
#                grpc_server_handling_seconds_bucket{
#                  namespace="tidb-management-service",
#                  job="tidb-mgmt-service"
#                }[5m]
#              )
#            )
#          ) > 0.30
#        for: 10m
#        labels:
#          severity: major
#          component: tidb-management-service
#        annotations:
#          description: p95 > {{ printf "%.0f" ($value * 1000) }} ms for {{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
#
#      - alert: gRPCServerP99LatencyHigh
#        expr: |
#          histogram_quantile(
#            0.99,
#            sum by (grpc_service, grpc_method, le) (
#              rate(
#                grpc_server_handling_seconds_bucket{
#                  namespace="tidb-management-service",
#                  job="tidb-mgmt-service"
#                }[5m]
#              )
#            )
#          ) > 1
#        for: 5m
#        labels:
#          severity: critical
#          component: tidb-management-service
#        annotations:
#          description: p99 > {{ printf "%.0f" ($value * 1000) }} ms for {{ $labels.grpc_service }}/{{ $labels.grpc_method }}.