groups:
  - name: serverless-service
    rules:
      - alert: ServerlessSvcNoAvailablePods
        expr: sum(kube_deployment_status_replicas_available{namespace=~".*-ms",deployment=~"serverless-svc-.*"}) by(deployment) == 0
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: There are no running pods of deployment {{ $labels.deployment }}.
      - alert: ServerlessSvcPodRestart
        expr: sum(increase(kube_pod_container_status_restarts_total{namespace=~".*-ms",pod=~"serverless-svc-.*"}[1m])) by(pod) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: The pod {{ $labels.pod }} has restarted.
      - alert: ServerlessSvcManyFailedHttpRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~"(serverless.ServerlessBillingService)",grpc_code!~"(OK|NotFound|InvalidArgument)"}[1m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~"(serverless.ServerlessBillingService)"}[1m])) by (grpc_service, grpc_method)
          ) > 0.1
        for: 10m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: HTTP requests failed many times({{ printf "%.2f" $value }}), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcTooManyFailedHttpRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~"(serverless.ServerlessBillingService)",grpc_code!~"(OK|NotFound|InvalidArgument)"}[1m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~"(serverless.ServerlessBillingService)"}[1m])) by (grpc_service, grpc_method)
          ) > 0.9
        for: 20m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: HTTP requests failed too many times({{ printf "%.2f" $value }}), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcManyFailedRpcRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~"(serverless.ServerlessBillingService)",grpc_code!~"(OK|NotFound|InvalidArgument)"}[1m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~"(serverless.ServerlessBillingService)"}[1m])) by (grpc_service, grpc_method)
          ) > 0.1
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: RPC requests failed too many times({{ printf "%.2f" $value }}), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcTooManyFailedRpcRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~"(serverless.ServerlessBillingService)",grpc_code!~"(OK|NotFound|InvalidArgument)"}[1m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~"(serverless.ServerlessBillingService)"}[1m])) by (grpc_service, grpc_method)
          ) > 0.5
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: RPC requests failed too many times({{ printf "%.2f" $value }}), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcTooManySlowRpcRequests
        expr: |
          histogram_quantile(0.99,
            sum(rate(grpc_server_handling_seconds_bucket{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_type="unary"}[1m])) by (grpc_service, grpc_method, le)
          ) > 5
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: RPC response latency P99 is {{ $value }}s, method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcDBQueryFailed
        expr: |
          sum(increase(serverless_gorm_errors_total{namespace=~".*-ms",error!="record not found"}[1m])) by (table, error) > 0
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: DB query failed, table={{ $labels.table }}, error={{ $labels.error }}.
      - alert: ServerlessSvcCapacityCheckpointDelay
        expr: sum(timestamp(serverless_instance_capacity_checkpoint_timestamp) - (serverless_instance_capacity_checkpoint_timestamp > 0)) by (pod) > 5400
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: serverless-svc
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: The capacity checkpoint of pod {{ $labels.pod }} has been delayed for {{ $value }}s.
      - alert: ServerlessSvcToolsManyFailedHttpRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.05
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: import
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: HTTP requests failed many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcToolsTooManyFailedHttpRequests
        # Exclude method ListDatabases|ListTargetTables because there are some unhandled but recoverable errors (e.g., accessDenied), and since the API does not raise critical warnings, the risk is relatively low.
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_method!~"(ListDatabases|ListTargetTables)",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.1
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: import
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: HTTP requests failed too many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcToolsManyFailedRpcRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.05
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: import
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: RPC requests failed too many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcToolsTooManyFailedRpcRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service=~".*(PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.1
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: import
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: RPC requests failed too many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcEcosystemManyFailedHttpRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.05
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: HTTP requests failed many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcEcosystemTooManyFailedHttpRequests
        # Exclude method ListDatabases|ListTargetTables because there are some unhandled but recoverable errors (e.g., accessDenied), and since the API does not raise critical warnings, the risk is relatively low.
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_method!~"(ListDatabases|ListTargetTables)",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-http-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.1
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: HTTP requests failed too many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcEcosystemManyFailedRpcRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.05
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: RPC requests failed too many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.
      - alert: ServerlessSvcEcosystemTooManyFailedRpcRequests
        expr: |
          (sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)",grpc_code!~"(OK|NotFound|InvalidArgument|AlreadyExists|Aborted|PermissionDenied|Unauthenticated)"}[5m])) by (grpc_service, grpc_method)
            / sum(increase(grpc_server_handled_total{namespace=~".*-ms",service=~"serverless-svc-grpc-metrics",grpc_service!~".*(ServerlessBillingService|PrivateLinkConnectionService|ImportService|ExportService|MigrationService|ChangefeedService)"}[5m])) by (grpc_service, grpc_method)
          ) > 0.1
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: critical
          response_class: page
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: RPC requests failed too many times({{ printf "%.2f" $value }}%), method={{ $labels.grpc_service }}/{{ $labels.grpc_method }}.