groups:
- name: lightning
  rules:
  - alert: TiKVStoreSizeImbalanceDuringImport
    annotations:
      summary: 'Sizes of TiKV stores are imbalance during import'
      message: TiKV sizes of Tenant {{ $labels.tenant}}, Cluster {{ $labels.namespace }} are imbalance during import for 5 longer than minutes.
    # TODO: adjust the alert ratio according to the real situation of production env
    expr: |
      (
        (stddev(sum(tikv_engine_size_bytes{cluster_id=~"tidb.*"}) by (instance, tenant, cluster_id)) by (tenant, cluster_id))
        /
        (avg(sum(tikv_engine_size_bytes{cluster_id=~"tidb.*"}) by (instance, tenant, cluster_id)) by (tenant, cluster_id))
        # coefficient of variation
        > 0.2
      ) and (
        # lightning job is running
        sum(lightning_engines{cluster_id=~"tidb.*"}) by (tenant, cluster_id)
      )
    for: 5m
    labels:
      component: lightning
      oncall_team: pd
      severity: major
      response_class: ticket