groups:
- name: TiDBCloudIncident
  rules:
  - alert: IncidentChangefeedHighDelay
    expr: ticdc_owner_checkpoint_ts_lag{} > 4*60*60 and on (tenant, project, tidb_cluster_id, changefeed) ticdc_owner_checkpoint_ts_lag{} offset 1h
    for: 1m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
        The Changefeed {{ $labels.changefeed }} of cluster {{ $labels.cluster_id }} checkpoint delay is more than 4 hours.
  - alert: PerformanceDegradeRisk
    expr: |
      (
        avg_over_time(sum by (cluster_id) (rate(tidb_executor_statement_total{tenant!~".*1372813089209061633"}[1m]))[5m:])
        < bool on (cluster_id) 0.2 * avg_over_time(
            sum by (cluster_id) (rate(tidb_executor_statement_total{tenant!~".*1372813089209061633"}[1m]))[1d:]
        )
      ) + on (cluster_id) (
        avg_over_time(histogram_quantile(0.99, sum by (cluster_id, le) (rate(tidb_server_handle_query_duration_seconds_bucket{tenant!~".*1372813089209061633"}[1m])))[5m:])
          > bool on (cluster_id) 100 * 
              histogram_quantile(0.99, sum by (cluster_id, le) (rate(tidb_server_handle_query_duration_seconds_bucket{tenant!~".*1372813089209061633"}[1d])))
      ) + on (cluster_id) (
        (
          avg_over_time(sum by (cluster_id) (increase(tidb_server_execute_error_total{type=~".*8027|.*8120|.*8021|.*8024|.*9001|.*9002|.*9003|.*9011|.*9012|.*9013",tenant!~".*1372813089209061633"}[1m]))[5m:]) 
          > bool 1
        ) or vector(0)
      ) >= 2
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
         The TiDB cluster {{ $labels.cluster_id }} has performance degrade risk. This indicates that TiDB's QPS has dropped by at least 80%, and the p99 latency has increased by at least 100 times.
  - alert: IncidentClusterOfflineStoreProgressPending
    annotations:
      message: The TiDB Cluster {{ $labels.cluster_id }} offline store progress pending for more than 1 hour
      runbook_url: https://pingcap.feishu.cn/wiki/Uctrwp42iirTGak0Ovzca9ldnkd?from=from_copylink
    expr: |
      increase(pd_cluster_progress{action="removing"}[2m]) == 0
    for: 1h
    labels:
      severity: critical
      response_class: page
      component: resilience
      oncall_team: resilience
  - alert: CertikTikvDiskReadIopsHigh
    expr: |
      (
        rate(node_disk_reads_completed_total[2m]) 
        and on(instance) 
        sum by(instance) (label_replace(kube_pod_info{namespace=~".*1379661944636804070", pod=~"db-tikv.*"}, "instance", "$1", "node", "(.+)"))
      ) > 3500
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
        The TiKV on {{ $labels.instance }} disk Read IOPS is too high.
  - alert: CertikTikvDiskWriteIopsHigh
    expr: |
      (
        rate(node_disk_writes_completed_total[2m]) 
        and on(instance) 
        sum by(instance) (label_replace(kube_pod_info{namespace=~".*1379661944636804070", pod=~"db-tikv.*"}, "instance", "$1", "node", "(.+)"))
      ) > 3500
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
        The TiKV on {{ $labels.instance }} disk Write IOPS is too high.
  - alert: TiflashMaxSnapshotLifetimeReachingLong
    expr: tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime > 18000
    for: 4h
    labels:
      severity: major
      response_class: ticket
      tier: dedicated
      component: resilience
      oncall_team: resilience
    annotations:
      description: The corresponding SQL query involves fetching a large volume of data with time spent on 'send response time'. Doubt to be a potential issue with client-side response handling the result.
      summary: "Long Running Tiflash Job"
  - alert: NetworkLagIncreaseForWetech
    expr: |
      sum(rate(tidb_tikvclient_rpc_net_latency_seconds_sum{tenant="1372813089209061633", scope="false"}[2m])) by (tenant, cluster_id, le, store) 
      / sum(rate(tidb_tikvclient_rpc_net_latency_seconds_count{tenant="1372813089209061633", scope="false"}[2m])) by (tenant, cluster_id, le, store)
      > 0.005
    for: 30s
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
        The network duration of store-{{ $labels.store }} in TiDB cluster {{ $labels.cluster_id }} is over 5ms.
  - alert: TiDBClusterAlertStorm
    expr: |
      (
        max_over_time(
          (
            count by (cluster_id) (
              ALERTS{
                alertstate="firing",
                severity=~"critical|major",
                alertname!="TiDBClusterAlertStorm"
              }
            )
          )[5m:]
        ) > 5
      )
      and on (cluster_id)
      (
        max_over_time(
          (
            count by (cluster_id) (
              ALERTS{
                alertstate="firing",
                severity=~"critical|major",
                alertname!="TiDBClusterAlertStorm"
              }
            )
          )[5m:]
        )
        >= 1.5 * (
          max_over_time(
            (
              count by (cluster_id) (
                ALERTS{
                  alertstate="firing",
                  severity=~"critical|major",
                  alertname!="TiDBClusterAlertStorm"
                }
              )
            )[5m:] offset 5m
          )
          or on (cluster_id)
          0 * max_over_time(
            (
              count by (cluster_id) (
                ALERTS{
                  alertstate="firing",
                  severity=~"critical|major",
                  alertname!="TiDBClusterAlertStorm"
                }
              )
            )[5m:]
          )
        )
      )
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
        The number of firing critical or major alerts in TiDB cluster {{ $labels.cluster_id }} exceeds 5 and has increased by at least 50% compared with the previous 5 minutes.

#  alert: WetechQPSRaised move to rules/dedicated/swat/data-plane/kernel-tcocp.yaml