groups:
  - name: gateway.rules
    rules:
      - alert: TiDBGatewayIsDown
        annotations:
          message: serverless gateway {{ $labels.job }} is down.
        expr: |
          up{provider_type=~"aws-free-tier|alicloud-serverless", job=~"gateway-metrics(.*)|privatelink-metrics", pod!~"haproxy(.*)|gateway-controller(.*)"} == 0
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TiDBGatewayConnectManagerFailed
        annotations:
          message: 'gataway {{ $labels.job }} connect manager failed, total count: {{ $value }}.'
        expr: |
          sum(increase(gateway_connection_total{stage="fetch_manager_failed"}[5m])) > 10
        for: 10m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TiDBGatewayUserEstablishConnectionTooLong
        annotations:
          message: user connect {{ $labels.job }} tidb pod with {{ $value }} seconds, more then 5s.
        expr: |
          histogram_quantile(0.95, sum(rate(gateway_connection_seconds_bucket{stage="establish_total", product="serverless",job!="gateway-metrics"}[10m])) by (le, product, stage)) > 5
        for: 10m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TiDBGatewayP95HandshakeTooLong
        annotations:
          message: '{{ $labels.job }} P95 time to connect to TiDB server for {{ $value }} seconds, more then 600ms.'
        expr: |
          histogram_quantile(0.95, sum(rate(gateway_connection_seconds_bucket{stage="connect_backend",job!="gateway-metrics"}[5m])) by (job, le)) > 0.6
        for: 5m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessPaidPoolIsBreakingSLA
        annotations:
          message: tidb_pool={{$labels.tidb_pool}}, gateway={{ $labels.pod }} SLA is {{ printf "%.3f" $value }}%, lower than 99.9% for 1m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, pod) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*",cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, pod))) * 100 < 99.9
        for: 1m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessPaidPoolIsCriticallyBreakingSLA
        annotations:
          message: tidb_pool={{$labels.tidb_pool}}, gateway={{ $labels.pod }} SLA is {{ printf "%.3f" $value }}%, lower than 90% for 5m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, pod) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*",cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, pod))) * 100 < 90
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessFreePoolIsBreakingSLA
        annotations:
          message: tidb_pool={{$labels.tidb_pool}}, gateway={{ $labels.pod }} SLA is {{ printf "%.3f" $value }}%, lower than 50% for 1m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (1 - (sum(increase(gateway_connection_sla_total{tidb_pool!~".*vip.*",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[10m])) by (tidb_pool, pod) /
          sum(increase(gateway_connection_sla_total{tidb_pool!~".*vip.*",cluster_id!="",status!="missmatch"}[10m])) by (tidb_pool, pod))) *100 < 50
        for: 1m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: StarterFreeClusterIsBreakingSLA
        annotations:
          message: cluster {{ $labels.pod }}/{{$labels.tidb_pool}}/{{ $labels.cluster_id }} SLA is {{ printf "%.3f" $value }}%, lower than 50% for 10m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (
            (1 - (
              sum(increase(gateway_connection_sla_total{tidb_pool=~"tidb-pool|canary-tidb-pool",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, pod, cluster_id)
              /
              sum(increase(gateway_connection_sla_total{tidb_pool=~"tidb-pool|canary-tidb-pool",cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, pod, cluster_id)
            )) * 100 < 50
          )
          and on(tidb_pool)
          (
            count(
              (1 - (
                sum(increase(gateway_connection_sla_total{tidb_pool=~"tidb-pool|canary-tidb-pool", cluster_id!="",status=~"inner_error|tidb_error|cancel"}[2m])) by (tidb_pool, cluster_id)
                /
                sum(increase(gateway_connection_sla_total{tidb_pool=~"tidb-pool|canary-tidb-pool", cluster_id!="",status!="missmatch"}[2m])) by (tidb_pool, cluster_id)
              )) < 0.50
            ) by (tidb_pool) < 5
          )
        for: 10m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: StarterPaidClusterIsCriticallyBreakingSLA
        annotations:
          message: cluster {{ $labels.pod }}/{{$labels.tidb_pool}}/{{ $labels.cluster_id }} SLA is {{ printf "%.3f" $value }}%, lower than 90% for 5m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (
            (1 - (
              sum(increase(gateway_connection_sla_total{tidb_pool=~"vip-tidb-pool|canary-vip-tidb-pool",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, pod, cluster_id)
              /
              sum(increase(gateway_connection_sla_total{tidb_pool=~"vip-tidb-pool|canary-vip-tidb-pool",cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, pod, cluster_id)
            )) * 100 < 90
          )
          and on(tidb_pool)
          (
            count(
              (1 - (
                sum(increase(gateway_connection_sla_total{tidb_pool=~"vip-tidb-pool|canary-vip-tidb-pool", cluster_id!="",status=~"inner_error|tidb_error|cancel"}[2m])) by (tidb_pool, cluster_id)
                /
                sum(increase(gateway_connection_sla_total{tidb_pool=~"vip-tidb-pool|canary-vip-tidb-pool", cluster_id!="",status!="missmatch"}[2m])) by (tidb_pool, cluster_id)
              )) < 0.90
            ) by (tidb_pool) < 5
          )
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: EssentialClusterIsCriticallyBreakingSLA
        annotations:
          message: cluster {{ $labels.pod }}/{{$labels.tidb_pool}}/{{ $labels.cluster_id }} SLA is {{ printf "%.3f" $value }}%, lower than 50% for 2m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*super-vip-tidb-pool",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, pod, cluster_id) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*super-vip-tidb-pool",cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, pod, cluster_id))) * 100 < 50
        for: 2m
        labels:
          tier: serverless
          severity: major
          response_class: ticket
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: EssentialClusterIsCriticallyBreakingSLA
        annotations:
          message: cluster {{ $labels.pod }}/{{$labels.tidb_pool}}/{{ $labels.cluster_id }} SLA is {{ printf "%.3f" $value }}%, lower than 90% for 5m. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          (1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*super-vip-tidb-pool",cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, pod, cluster_id) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*super-vip-tidb-pool",cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, pod, cluster_id))) * 100 < 90
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TooManyClustersAreBreakingSLA
        annotations:
          message: There are {{ $value }} clusters in {{$labels.tidb_pool}} are breaking the SLA. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          count((1 - (sum(increase(gateway_connection_sla_total{tidb_pool!~".*vip.*", cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, cluster_id) /
          sum(increase(gateway_connection_sla_total{tidb_pool!~".*vip.*", cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, cluster_id))) < 0.90) by (tidb_pool)
        for: 5m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TooManyClustersAreBreakingSLA
        annotations:
          message: There are {{ $value }} clusters in {{$labels.tidb_pool}} are breaking the SLA. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          count((1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*", cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, cluster_id) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*", cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, cluster_id))) < 0.99) by (tidb_pool) <= 3
        for: 2m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TooManyClustersAreBreakingSLA
        annotations:
          message: There are {{ $value }} clusters in {{$labels.tidb_pool}} are breaking the SLA. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          count((1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*", cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, cluster_id) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*", cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, cluster_id))) < 0.99) by (tidb_pool) > 3
        for: 2m
        labels:
          tier: serverless
          severity: Major
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TooManyClustersAreBreakingSLA
        annotations:
          message: There are {{ $value }} clusters in {{$labels.tidb_pool}} are breaking the SLA. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now) / [Debug Tracing](https://clinic.pingcap.com/grafana/d/GhRSdFSIz/serverless-cluster-debug-tracing?orgId=1&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-cluster_id=10105029606211911639&var-keyspace_id=59381&var-keyspace_name=41b5EeRg1sbiv2A&var-instance=All&from=now-1h&to=now&refresh=30s)
        expr: |
          count((1 - (sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*", cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, cluster_id) /
          sum(increase(gateway_connection_sla_total{tidb_pool=~".*vip.*", cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, cluster_id))) < 0.99) by (tidb_pool) > 3
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessPocAction
        annotations:
          message: cluster {{ $labels.service }} / {{ $labels.tidb_pool }} / {{ $labels.cluster_id }} has {{ $value }} concurrent connections, may be doing a PoC, please pay attention.
        expr: |
          sum(increase(gateway_connection_sla_total{cluster_id!=""}[1m])) by (cluster_id, tidb_pool, service) > 2000
        for: 1h
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessIsBreakingSLAorMetricMissing
        annotations:
          message: serverless SLA is {{ $value }}, lower than 0.5. [SLA Board](https://clinic.pingcap.com/grafana/d/8fh_Hs-4k/serverless-sla-board?orgId=1&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-mode=All&var-instance=All&var-cluster_id=All&from=now-1h&to=now)
        expr: |
          (1 - ((sum(increase(gateway_connection_sla_total{cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) or vector(1)) / (sum(increase(gateway_connection_sla_total{cluster_id!="",status!="missmatch"}[1m])) or vector(1)))) < 0.50
        for: 10m
        labels:
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: TiDBGatewayHighCPU
        expr: (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace="tidb-admin",container="tidb-gateway"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="tidb-admin",container="tidb-gateway",resource="cpu"}) by (pod))*100 > 70
        for: 2m
        labels:
          severity: critical
          response_class: page
          tier: serverless
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
        annotations:
          message: 'CPU utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 70% for 2m'
          runbook_url:
          value: '{{ $value }}'
          summary: CPU usage exceeds the threshold
      - alert: TiDBGatewayHighMemory
        expr: (sum(container_memory_working_set_bytes{namespace="tidb-admin",container="tidb-gateway",container!="POD", container!="", image!=""}) by (pod) / sum(kube_pod_container_resource_limits{namespace="tidb-admin",container="tidb-gateway",resource="memory"}) by (pod))*100 > 70
        for: 2m
        labels:
          severity: critical
          response_class: page
          tier: serverless
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
        annotations:
          message: 'Memory utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 70% for 2m'
          runbook_url:
          value: '{{ $value }}'
          summary: Memory usage exceeds the threshold