groups:
# NOTE on the `tenant` label below:
# jiraport's `ue-cassandra-external` customer route filters on `tenant_id`
# (customfield_10157), which is populated from the alert's `tenant` label. Without a
# `tenant` label the external notification is dropped. The proxy metrics carry the
# tenant id on the `tenant_id` series label, so we keep it through the aggregation
# (`sum by (ClusterId, tenant_id, ...)`) and re-emit it as `tenant`. This works for any
# number of customers without hardcoding and needs no jiraport change.
- name: cassandra-on-tikv-customer
  rules:
  - alert: CassandraOnTiKVHighRequestErrorRate
    expr: |
      (
        100 * sum by (ClusterId, tenant_id) (
          rate(proxy_request_completed_total{status="error"}[2m])
        )
        /
        clamp_min(
          sum by (ClusterId, tenant_id) (
            rate(proxy_request_completed_total[2m])
          ),
          0.001
        )
      ) > 1.5
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_request_total[2m])
      ) > 5
    for: 2m
    labels:
      tier: next-gen
      severity: major
      component: cassandra
      visibility: external
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: '[{{ $labels.cluster_id }}] Cassandra-on-TiKV request error rate exceeded 1.5%'
      description: >-
        Instance {{ $labels.cluster_id }} request error rate is {{ printf "%.2f" $value }}%
        (threshold: > 1.5%), sustained for 2 minutes with traffic > 5 req/s.
        
        Suggested actions: 
        (1) Check your application logs for specific error types;
        (2) Locate failed type through 'Failed Operations' metrics in the TiDB Cloud console;
        (3) Contact TiDB Cloud support.
  - alert: CassandraOnTiKVHighRequestP99Latency
    expr: |
      histogram_quantile(
        0.99,
        sum by (ClusterId, tenant_id, le) (
          rate(proxy_request_duration_seconds_bucket{status="ok"}[2m])
        )
      ) > 0.5
      and on (ClusterId)
      sum by (ClusterId) (
        rate(proxy_request_total[2m])
      ) > 5
    for: 2m
    labels:
      tier: next-gen
      severity: major
      component: cassandra
      visibility: external
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: '[{{ $labels.cluster_id }}] Cassandra-on-TiKV overall P99 request latency exceeded 500ms'
      description: >-
        Instance {{ $labels.cluster_id }} overall P99 request latency is {{ printf "%.3f" $value }}s
        (threshold: > 500ms), sustained for 2 minutes with traffic > 5 req/s.
        
        Suggested actions: 
        (1) Locate high latency request type through 'Operation Latency' metrics in the TiDB Cloud console;
        (2) Contact TiDB Cloud support if latency remains elevated.
  - alert: CassandraOnTiKVHighSelectRequestP99Latency
    expr: |
      histogram_quantile(
        0.99,
        sum by (ClusterId, tenant_id, le) (
          rate(proxy_request_duration_seconds_bucket{status="ok", op_type="select"}[2m])
        )
      ) > 0.2
      and on (ClusterId)
      sum by (ClusterId) (
        increase(proxy_request_completed_total{status="ok", op_type="select"}[2m])
      ) >= 100
    for: 2m
    labels:
      tier: next-gen
      severity: major
      component: cassandra
      visibility: external
      cluster_id: '{{ $labels.ClusterId }}'
      tenant: '{{ $labels.tenant_id }}'
    annotations:
      summary: '[{{ $labels.cluster_id }}] Cassandra-on-TiKV SELECT P99 latency exceeded 200ms'
      description: >-
        Instance {{ $labels.cluster_id }} SELECT request P99 latency is {{ printf "%.3f" $value }}s
        (threshold: > 200ms), with ≥ 100 successful SELECT requests in the last 2 minutes.
        
        Suggested actions: 
        (1) Check your application logs for 'SELECT' CQLs;
        (2) Locate latency trend through 'Operation Latency' metrics in the TiDB Cloud console;
        (3) Contact TiDB Cloud support if latency remains elevated.