groups:
- name: cassandra-on-tikv-internal
  rules:
  - alert: CassandraOnTiKVHighRequestErrorRateInternal
    expr: |
      (
        100 * sum by (ClusterId, tenant_id) (
          rate(proxy_request_completed_total{status="error"}[1m])
        )
        /
        clamp_min(
          sum by (ClusterId, tenant_id) (
            rate(proxy_request_completed_total[1m])
          ),
          0.001
        )
      ) > 1
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_request_total[1m])
      ) > 5
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv request error rate is high
      description: 'cluster: {{ $labels.cluster_id }}, request error rate: {{ $value }}%'
  - alert: CassandraOnTiKVHighRequestP99LatencyInternal
    expr: |
      histogram_quantile(
        0.99,
        sum by (ClusterId, tenant_id, le) (
          rate(proxy_request_duration_seconds_bucket{status="ok"}[1m])
        )
      ) > 0.5
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_request_total[1m])
      ) > 5
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv request p99 latency is high
      description: 'cluster: {{ $labels.cluster_id }}, request p99 latency: {{ $value }}s'
  - alert: CassandraOnTiKVHighRequestP999LatencyInternal
    expr: |
      histogram_quantile(
        0.999,
        sum by (ClusterId, tenant_id, le) (
          rate(proxy_request_duration_seconds_bucket{status="ok"}[1m])
        )
      ) > 1
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_request_total[1m])
      ) > 5
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv request p99.9 latency is high
      description: 'cluster: {{ $labels.cluster_id }}, request p99.9 latency: {{ $value }}s'
  - alert: CassandraOnTiKVHighSelectRequestP99LatencyInternal
    expr: |
      histogram_quantile(
        0.99,
        sum by (ClusterId, tenant_id, le) (
          rate(proxy_request_duration_seconds_bucket{status="ok", op_type="select"}[1m])
        )
      ) > 0.2
      and on (ClusterId)
      sum by (ClusterId) (
        increase(proxy_request_completed_total{status="ok", op_type="select"}[1m])
      ) >= 100
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv select request p99 latency is high
      description: 'cluster: {{ $labels.cluster_id }}, select request p99 latency: {{ $value }}s'
  - alert: CassandraOnTiKVHighRequestStageP99Latency
    expr: |
      histogram_quantile(
        0.99,
        sum by (ClusterId, tenant_id, stage, le) (
          rate(proxy_request_stage_seconds_bucket[5m])
        )
      ) > 0.5
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_request_total[5m])
      ) > 10
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv request stage p99 latency is high
      description: 'cluster: {{ $labels.cluster_id }}, stage: {{ $labels.stage }}, stage p99 latency: {{ $value }}s'
  - alert: CassandraOnTiKVHighBackendErrorRate
    expr: |
      (
        100 * sum by (ClusterId, tenant_id) (
          rate(proxy_mysql_error_total[5m])
        )
        /
        clamp_min(
          sum by (ClusterId, tenant_id) (
            rate(proxy_mysql_operation_total[5m])
          ),
          0.001
        )
      ) > 1
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_mysql_operation_total[5m])
      ) > 10
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv backend error rate is high
      description: 'cluster: {{ $labels.cluster_id }}, backend error rate: {{ $value }}%'
  - alert: CassandraOnTiKVBackendPoolSaturated
    expr: |
      (
        sum by (ClusterId, tenant_id) (proxy_mysql_pool_in_use)
        /
        clamp_min(
          sum by (ClusterId, tenant_id) (proxy_mysql_pool_open),
          1
        )
      ) > 0.8
      and on (ClusterId)
      sum by (ClusterId) (
        increase(proxy_mysql_pool_wait_total[5m])
      ) > 100
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv backend pool is saturated
      description: 'cluster: {{ $labels.cluster_id }}, backend pool utilization: {{ $value }}'
  - alert: CassandraOnTiKVInFlightSpike
    expr: |
      min_over_time((
        sum by (ClusterId, instance) (
          proxy_request_in_flight
        )
      )[1m:]) > 20
      and (
        (
          max_over_time((
            sum by (ClusterId, instance) (
              proxy_request_in_flight
            )
          )[1m:])
          -
          min_over_time((
            sum by (ClusterId, instance) (
              proxy_request_in_flight
            )
          )[1m:])
        )
        /
        clamp_min(
          min_over_time((
            sum by (ClusterId, instance) (
              proxy_request_in_flight
            )
          )[1m:]),
          1
        )
      ) > 0.1
    for: 1m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: cassandra
      visibility: internal
      assign: zhi.xie@pingcap.com
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: cassandra-on-tikv in-flight requests are spiking on one instance
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, pod: {{ $labels.pod }}, in-flight requests stayed above 20 for 60s and are still rising'
      runbook_url: https://github.com/tidbcloud/runbooks/wiki/CassandraOnTiKVInFlightSpike