groups:
  - name: subscribe.mgmt.serverless-global-service
    rules:
      - alert: RequestRegionalServerFailed
        expr: |
          sum(rate(global_serverless_control_regional_request_failed_total[1m])) by (path) > 1
        for: 1m
        labels:
          severity: major
          response_class: ticket
          server: serverless-global-service
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: request regional server failed. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)

      - alert: TooManyClusterInitializedFailed
        expr: |
          sum(increase(global_serverless_control_cluster_initialized_failed_total[1m])) > 300
        for: 3m
        labels:
          severity: major
          response_class: ticket
          server: serverless-global-service
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: there are {{ $value }} clusters initialize failed for 3m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)

      - alert: TooManyRegionalRequests
        expr: |
          sum(rate(global_serverless_control_regional_request_total[1m])) by (path) 
            > 180
        for: 1m
        labels:
          severity: major
          response_class: ticket
          server: serverless-global-service
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: too many regional request. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)

      - alert: TooSlowRegionalResponse
        expr: |
          histogram_quantile(0.99, sum(rate(global_serverless_control_regional_request_time_seconds_bucket{namespace=~"tidb-management-service"}[1m])) by (path, method, le))
            > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          server: serverless-global-service
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: regional response too slow. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)

      - alert: TooManyClusterIDInRequest
        expr: |
          sum(rate(global_serverless_control_regional_request_too_many_clusterid[1m]))
          > 0
        for: 1s
        labels:
          severity: major
          response_class: ticket
          server: serverless-global-service
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: too many cluster ids in request. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1)

      - alert: TooManyClusterDeleteOrUpdateRequests
        expr: |
          sum(increase(tidb_management_service_serverless_cluster_api_write_request_success_total{api=~"delete_cluster|update_cluster"}[1h])) by (api, tenant_id)
            > 100
        labels:
          severity: major
          response_class: ticket
          server: serverless-global-service
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: tenant {{ $labels.tenant_id }} has {{ $value }} {{ $labels.api }} requests in the last hour (>100). [Diagnosis Board](https://clinic.pingcap.com/grafana/d/0hqVrAVvz/serverless-global-service-monitor?orgId=1&from=now-24h&to=now&viewPanel=46)