groups:
- name: serverless-customer-bu
  rules:
  # BU customer-owned alert parity rules. These rules mirror alert conditions
  # defined by the customer and are scoped to tenant_id 1372813089209316213.
  - alert: TiDBCustomerSlowQueriesHighInternal
    expr: |-
      (
        sum by (cluster_id, tenant_id, provider_type) (
          rate(tidb_server_handle_query_duration_seconds_count{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])
        )
        -
        sum by (cluster_id, tenant_id, provider_type) (
          rate(tidb_server_handle_query_duration_seconds_bucket{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal", le="1"}[2m])
        )
      ) > 5
    for: 5m
    labels:
      tier: serverless
      severity: major
      response_class: ticket
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, slow query rate over 1s: {{ $value }}/s'
      summary: TiDB slow query rate exceeds 5/s for tenant 1372813089209316213
  - alert: TiDBCustomerLockConflictHighInternal
    expr: sum(rate(tidb_server_execute_error_total{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", type=~"kv:9007|kv:8005"}[5m])) by (cluster_id, tenant_id, provider_type) > 3
    for: 5m
    labels:
      tier: serverless
      severity: major
      response_class: ticket
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, lock/write conflict rate: {{ $value }}/s'
      summary: TiDB lock conflict rate exceeds 3/s for tenant 1372813089209316213
  - alert: TiDBCustomerLockWaitTimeHighInternal
    expr: |-
      histogram_quantile(
        0.99,
        sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", type="acquire_pessimistic_lock"}[2m])) by (cluster_id, le)
      ) > 3
    for: 5m
    labels:
      tier: serverless
      severity: major
      response_class: ticket
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, pessimistic lock wait p99: {{ $value }}s'
      summary: TiDB pessimistic lock wait P99 exceeds 3 seconds for tenant 1372813089209316213
  - alert: TiDBCustomerTransactionAbortHighInternal
    expr: sum(rate(tidb_server_execute_error_total{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", type=~"kv:9007|kv:8005|tikv:9006|tikv:1205"}[5m])) by (cluster_id, tenant_id, provider_type) > 10
    for: 5m
    labels:
      tier: serverless
      severity: major
      response_class: ticket
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, transaction abort or conflict error rate: {{ $value }}/s'
      summary: TiDB transaction abort or conflict rate exceeds 10/s for tenant 1372813089209316213
  - alert: TiDBCustomerQueryDurationP99HighInternal
    expr: |-
      histogram_quantile(
        0.99,
        sum(rate(tidb_server_handle_query_duration_seconds_bucket{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, le)
      ) > 3
    for: 5m
    labels:
      tier: serverless
      severity: critical
      response_class: page
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, query duration p99: {{ $value }}s'
      summary: TiDB query duration P99 exceeds 3 seconds for tenant 1372813089209316213
  - alert: TiDBCustomerQueryDurationP95HighInternal
    expr: |-
      histogram_quantile(
        0.95,
        sum(rate(tidb_server_handle_query_duration_seconds_bucket{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, le)
      ) > 1
    for: 5m
    labels:
      tier: serverless
      severity: major
      response_class: ticket
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, query duration p95: {{ $value }}s'
      summary: TiDB query duration P95 exceeds 1 second for tenant 1372813089209316213
  - alert: TiDBCustomerDatabaseTimeHighInternal
    expr: sum(rate(tidb_server_handle_query_duration_seconds_sum{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type) > 20
    for: 5m
    labels:
      tier: serverless
      severity: major
      response_class: ticket
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, database time: {{ $value }}s/s'
      summary: TiDB database time exceeds 20 seconds per second for tenant 1372813089209316213
  - alert: TiDBCustomerAverageQueryDurationWarningInternal
    expr: |-
      (
        sum(rate(tidb_server_handle_query_duration_seconds_sum{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type)
        /
        sum(rate(tidb_server_handle_query_duration_seconds_count{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type)
      ) > 0.5
    for: 5m
    labels:
      tier: serverless
      severity: warning
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, average query duration: {{ $value }}s'
      summary: TiDB average query duration exceeds 0.5 seconds for tenant 1372813089209316213
  - alert: TiDBCustomerAverageQueryDurationPhoneInternal
    expr: |-
      (
        sum(rate(tidb_server_handle_query_duration_seconds_sum{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type)
        /
        sum(rate(tidb_server_handle_query_duration_seconds_count{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type)
      ) > 1
    for: 5m
    labels:
      tier: serverless
      severity: critical
      response_class: page
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, average query duration: {{ $value }}s'
      summary: TiDB average query duration exceeds 1 second for tenant 1372813089209316213
  - alert: TiDBCustomerAverageQueryDurationCriticalInternal
    expr: |-
      (
        sum(rate(tidb_server_handle_query_duration_seconds_sum{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type)
        /
        sum(rate(tidb_server_handle_query_duration_seconds_count{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) by (cluster_id, tenant_id, provider_type)
      ) > 3
    for: 5m
    labels:
      tier: serverless
      severity: critical
      response_class: page
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, average query duration: {{ $value }}s'
      summary: TiDB average query duration exceeds 3 seconds for tenant 1372813089209316213
  - alert: TiDBCustomerAverageDatabaseTimeHighInternal
    expr: avg by (cluster_id, tenant_id, provider_type) (rate(tidb_server_handle_query_duration_seconds_sum{tenant_id="1372813089209316213", provider_type=~"aws-free-tier|alicloud-serverless", component="tidb", sql_type!="internal"}[2m])) > 10
    for: 5m
    labels:
      tier: serverless
      severity: critical
      response_class: page
      component: tidb
      visibility: internal
      source_component: tidb
      oncall_team: none
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, tenant: {{ $labels.tenant_id }}, average database time by series: {{ $value }}s/s'
      summary: TiDB average database time exceeds 10 seconds per second for tenant 1372813089209316213