groups:
  - name: serverless-service-branch
    rules:
      - alert: BranchTooManyFailedRequests
        expr: |
          (sum(increase(serverless_branch_requested_total{namespace=~".*-ms", code=~"5.*"}[3m])) by (type,method)
            / sum(increase(serverless_branch_requested_total{namespace=~".*-ms"}[3m])) by (type,method)
          ) > 0.1
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: Serverless Branch {{ $labels.type }} {{ $labels.method }} has too may failed requests.

      - alert: BranchTooManySlowRequests
        expr: |
          histogram_quantile(0.99,
            sum(rate(serverless_branch_handling_seconds_bucket{namespace=~".*-ms"}[3m])) by (type, method, le)
          ) > 5
        for: 5m
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: branch response latency P99 is {{ $value }}s, method={{ $labels.type }}/{{ $labels.method }}.

      # adjust the threshold after test
      - alert: ServerlessBranchReconcileError
        expr: |
          sum(increase(serverless_branch_reconcile_err_total{namespace=~".*-ms"}[1m])) by (method, resource_id) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: serverless-svc-worker method {{ $labels.method }} error, branch {{ $labels.resource_id }}.

      # Limiter runs every 30 seconds
      - alert: ServerlessBranchLimiterError
        expr: |
          increase(serverless_branch_limit_process_statistics_total{namespace=~".*-ms", status="failed"}[5m]) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          # provider and region conflicts with server-side labels, see prometheus honor_labels config
          message: "Serverless Branch limiter {{ $labels.exported_provider }}/{{ $labels.exported_region }} error: {{ $labels.reason }}."

      - alert: BranchWaitReadyTimeout
        expr: |
          sum(serverless_branch_wait_ready_duration_seconds{namespace=~".*-ms"}) by (branch_id) > 600
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: Serverless Branch {{ $labels.branch_id }} wait ready time out.

      - alert: BranchHandleClusterEventError
        expr: |
          sum(increase(serverless_branch_sync_process_statistics_total{namespace=~".*-ms", status="failed"}[5m])) by (event_id) >= 5
        labels:
          tier: TiDB Serverless
          severity: warning
          component: ecosystem-server
          oncall_team: serverless
          provider_type: aws-free-tier
        annotations:
          message: Serverless Branch handle cluster event {{ $labels.event_id }} error.