groups:
- name: tikv-data-plane
  rules:
  #- alert: TiKVServerDowntimeIsTooLong
  #  expr: up{component="tikv", job=~".*tikv.*", job!=".*worker.*", } == 0
  #  for: 5m
  #  labels:
  #    tier: next-gen
  #    severity: major
  #    expr: up{component="tikv", job=~".*tikv.*", job!=".*worker.*", } == 0
  #    component: tikv
  #  annotations:
  #    description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #    runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
  #    value: '{{ $value }}'
  #    summary: TiKV server downtime is more than 5 min
  #  # move to kernel-governance.rules.yaml
  # - alert: TiKVServerDowntimeIsTooLong
  #   expr: up{component="tikv", job=~".*tikv.*", job!=".*worker.*", } == 0
  #   for: 10m
  #   labels:
  #     tier: next-gen
  #     severity: critical
  #     component: tikv
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
  #     value: '{{ $value }}'
  #     summary: TiKV server downtime is more than 10 min
  #     message: |
  #       TiKV server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has been down for more than 10 minutes. This affects data storage and may impact cluster performance and availability. Please immediately check and restart the TiKV server to restore data storage functionality.
  - alert: TiKVServerFrequentlyRestart
    expr: |-
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*tikv.*", job!=".*worker.*", }))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*tikv.*", job!=".*worker.*", }[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV server {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has restarted {{ $value }} times in the past hour. Frequent TiKV restarts can affect data storage and cluster stability. Please investigate the root cause and ensure TiKV server stability. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server is down for at least 3 times in a hour
  - alert: TiKVClusterFrequentlyRestart
    expr: |-
      (
        sum by (tidb_cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*tikv.*", job!=".*worker.*", }))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*tikv.*", job!=".*worker.*", }[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*tikv.*", job!=".*worker.*", }))[1h:15s])
            ) by (tidb_cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (tidb_cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'Multiple TiKV instances in cluster {{ $labels.tidb_cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting TiKV stability. Please investigate the cluster infrastructure and TiKV configuration to prevent further restarts. instance: {{ $labels.instance }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV instances for a cluster is down for at least 3 times in a hour
  # - alert: TiKVServerRestarted
  #   expr: sum(resets((-floor(process_start_time_seconds{job=~".*tikv.*", job!=".*worker.*", }))[1h:15s])) by (cluster_id, instance, tenant, provider_type) >= 1 and sum(max_over_time(up{job=~".*tikv.*", job!=".*worker.*", }[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
  #   for: 10m
  #   labels:
  #     tier: next-gen
  #     severity: warning
  #     component: tikv
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
  #     value: '{{ $value }}'
  #     summary: TiKV server is down for at least 1 times in a hour
  #     message: |
  #       TiKV server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has restarted {{ $value }} time(s) in the past hour. Please monitor the TiKV server for stability and investigate if restarts become frequent.
  # - alert: TiKVServerDownMoreThan1AZ
  #   expr: |
  #       count(count(
  #         label_replace((kube_pod_info{pod=~"db-tikv.*"} * on(node) group_left(provider_id) kube_node_info),"az_id",  "$1", "provider_id", "(.*/)(\\w+.*)")
  #         * on(pod,namespace) group_right(az_id)
  #         label_replace(label_replace(up{component="tikv"},"pod",  "$0", "instance", ".*"),"namespace",  "$0", "cluster_id", ".*") == 0
  #       ) by (cluster_id, az_id)) by (cluster_id) > 1
  #   for: 1m
  #   labels:
  #     severity: emergency
  #     component: tikv
  #     expr: |
  #         count(count(
  #           label_replace((kube_pod_info{pod=~"db-tikv.*"} * on(node) group_left(provider_id) kube_node_info),"az_id",  "$1", "provider_id", "(.*/)(\\w+.*)")
  #           * on(pod,namespace) group_right(az_id)
  #           label_replace(label_replace(up{component="tikv"},"pod",  "$0", "instance", ".*"),"namespace",  "$0", "cluster_id", ".*") == 0
  #         ) by (cluster_id, az_id)) by (cluster_id) > 1
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, down tikv servers which in different AZ, count: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ijox0linkuyj
  #     value: '{{ $value }}'
  #     summary: "TiKV server down count more than 1 and in different AZ"
  # - alert: TiKV_critical_error
  #   expr: sum(rate(tikv_critical_error_total[1m])) BY (cluster_id, instance, tenant, provider_type) > 0
  #   # without the for clause will become active on the first evaluation.
  #   labels:
  #     env: ENV_LABELS_ENV
  #     level: emergency
  #     expr: sum(rate(tikv_critical_error_total[1m])) BY (cluster_id, instance, tenant, provider_type) > 0
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?#heading=h.kq5ljh5ykmek
  #     value: '{{ $value }}'
  #     summary: "TiKV encounters critical errors"
  - alert: TiKVRaftstoreCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"(raftstore|rs|rfstore)_.*"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"(raftstore|rs|rfstore)_.*"}) by (tidb_cluster_id, instance, tenant, provider_type)  > 0.8
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Raftstore high CPU load (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV raftstore CPU usage in cluster {{ $labels.tidb_cluster_id }} on instance {{ $labels.instance }} has exceeded 80% for 5 minutes. High raftstore CPU usage may affect Raft consensus performance and cluster stability. Consider scaling up the TiKV node or optimizing the workload to reduce CPU pressure. value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ynohpbfagetz
  - alert: TiKVApplyWorkerPoolCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+|apply_follower_[0-9]+"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+|apply_follower_[0-9]+"}) by (tidb_cluster_id, instance, tenant, provider_type)  > 0.9
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Apply worker thread pool high CPU load (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV apply worker thread pool CPU usage in cluster {{ $labels.tidb_cluster_id }} on instance {{ $labels.instance }} has exceeded 90% for 5 minutes. High apply worker CPU usage may affect write performance and data consistency. Consider scaling up the TiKV node or optimizing the write workload. value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.7k1o575l3zea
  - alert: TiKVSchedulerWorkerCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"sched_.*"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"sched_.*"}) by (tidb_cluster_id, instance, tenant, provider_type)  > 0.9
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Scheduler worker high CPU load (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV scheduler worker CPU usage in cluster {{ $labels.tidb_cluster_id }} on instance {{ $labels.instance }} has exceeded 90% for 5 minutes. High scheduler worker CPU usage may affect transaction scheduling and performance. Consider scaling up the TiKV node or optimizing the transaction workload. TiKV node''s scheduler worker thread pool is under a high load VALUE = {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.qptrnffkw5h
  - alert: TiKVGrpcPollHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"grpc.*"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"grpc.*"}) by (tidb_cluster_id, instance, tenant, provider_type)  > 0.8
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "gRPC high CPU load (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV gRPC polling thread pool CPU usage in cluster {{ $labels.tidb_cluster_id }} on instance {{ $labels.instance }} has exceeded 80% for 5 minutes. High gRPC CPU usage may affect client request processing and cluster communication. Consider scaling up the TiKV node or optimizing client request patterns. value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.30ra2mgpp1i4
  # Covered by kernel-tcocp.rules
  # - alert: TiKVUnifiedReadPoolHighLoad
  #   expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"unified_read_po.*"}[1m])) by (cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"unified_read_po.*"}) by (cluster_id, instance, tenant, provider_type)  > 0.9
  #   for: 5m
  #   labels:
  #     tier: next-gen
  #     severity: warning
  #     component: tikv
  #   annotations:
  #     summary: "Unified read pool high CPU load (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
  #     description: "TiKV node's unified read pool is under a high load\n  VALUE = {{ $value }}"
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.y1jc473vb50t
  #     message: |
  #       TiKV unified read pool CPU usage in cluster {{ $labels.cluster_id }} on instance {{ $labels.instance }} has exceeded 90% for 5 minutes. High unified read pool CPU usage may affect read performance and query response times. Consider scaling up the TiKV node or optimizing read workload patterns.

  - alert: TiKVTooManyRaftLogDroppedCount
    expr: sum(rate(tikv_server_raft_append_rejects{type!="kv_gc"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type) > 100
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Too many raft log dropped, likely due to excessive memory usage (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV node {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} is dropping too many raft logs ({{ $value }} per minute) due to excessive memory usage. This may affect data consistency and cluster performance. Please check memory usage and consider scaling up the node or optimizing memory allocation.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVTooManyFailedGRPCRequests
    expr: rate(tikv_grpc_msg_fail_total{type!="kv_gc"}[1m]) > 100
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node has too many grpc requests failed (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV node {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has too many failed gRPC requests ({{ $value }} per minute). This may indicate network issues or server overload affecting client communication. Please check network connectivity and server performance.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftAppendLogSlow
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket[1m])) by (le, tidb_cluster_id, instance, tenant, provider_type)) > 30
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's raft append log P99 duration in seconds is too slow (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV node {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has slow raft append log performance with P99 duration of {{ $value }} seconds (threshold: 30s). This may affect write performance and cluster consensus. Please check disk I/O performance and consider optimizing storage or scaling up the node.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftApplyLogDurationSecs
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, tidb_cluster_id, instance, tenant, provider_type)) > 45
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's raft apply log P99 duration is too slow (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV node {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has slow raft apply log performance with P99 duration of {{ $value }} seconds (threshold: 45s). This may affect write performance and data consistency. Please check disk I/O performance and consider optimizing storage or scaling up the node.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  # - alert: TiKVReportTooManyFailureMsg
  #   expr:  sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[1m])) BY (cluster_id, instance, tenant, provider_type) > 100
  #   for: 5m
  #   labels:
  #     tier: next-gen
  #     severity: warning
  #     component: tikv
  #   annotations:
  #     summary: "TiKV node reported too many unreachable failure messages (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
  #     description: "TiKV node reported too many unreachable failure messages\n  VALUE = {{ $value }}"
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
  #     message: |
  #       TiKV node {{ $labels.instance }} in cluster {{ $labels.cluster_id }} is reporting too many unreachable failure messages ({{ $value }} per minute). This indicates network connectivity issues between TiKV nodes. Please check network connectivity and resolve any network partitioning issues.

  - alert: TiKVChannelFull
    expr: sum(rate(tikv_channel_full_total[1m])) BY (type, tidb_cluster_id, instance, tenant, provider_type) > 0
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's gRPC channel is full (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV node {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has full gRPC channels ({{ $value }} per minute). This indicates that the gRPC communication channels are saturated, which may affect client request processing. Please check server load and consider scaling up the node or optimizing request patterns.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftLogLagAbove5000For30mNextGen
    expr: |-
      (
        max by (tidb_cluster_id, tenant, provider_type) (histogram_quantile(1, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, tidb_cluster_id, instance, tenant, provider_type))) > 5000
      )
      and
      (
        sum by (tidb_cluster_id, tenant, provider_type) (pd_regions_status{type="down-peer-region-count"}) > 0
      )
    for: 30m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's Raft log lag is more than 5000 (cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }})"
      description: 'TiKV node {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has a Raft log lag of more than 5000 entries ({{ $value }}). This indicates that the node is falling behind in Raft log replication, which may affect data consistency. Please check node performance and network connectivity.'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVLargeCommitLatencyNode
    expr: max(histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (tidb_cluster_id, instance, tenant, provider_type))) >= 5 * quantile(0.5, histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (tidb_cluster_id, instance, tenant, provider_type)))
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV node in cluster {{ $labels.tidb_cluster_id }} has been identified as a slow node with commit log duration 5x higher than the median value. This may affect overall cluster performance and cause uneven load distribution. Please investigate the slow node and consider optimizing its performance or replacing it. value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      summary: "TiKV node reported a slow TiKV node---its commit log duration is 5x more of the median value (cluster: {{ $labels.tidb_cluster_id }})"

  - alert: TiKVBackupFailureFor1h
    expr: sum(increase(native_br_backup_success[1h])) == 0
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV auto backup has no successful run for 1 hour"
      description: 'TiKV auto backup in cluster {{ $labels.tidb_cluster_id }} has no successful run in the last hour. cluster: {{ $labels.tidb_cluster_id }}, values: {{ $value }}'

  - alert: TiKVInodeUsageMoreThan90%
    expr: |
      kubelet_volume_stats_inodes_used{persistentvolumeclaim=~".+-db-[a-z0-9]+-tikv-.+"}/kubelet_volume_stats_inodes{persistentvolumeclaim=~".+-db-[a-z0-9]+-tikv-.+"} > 0.9
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: next-gen
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV inode usage for PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} has exceeded 90% ({{ $value }}). High inode usage may prevent new file creation and affect TiKV operations. Please clean up unnecessary files or expand storage capacity to resolve the inode shortage.'
      value: '{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?tab=t.0#heading=h.w4zl0kl6ny53
      summary: "TiKV inode usage more than 90%"
  # Covered by kernel-tcocp.rules
  # - alert: MaxTiKVMemoryUsageTooHighInternal
  #   expr: |-
  #     max by (tidb_cluster_id) (
  #       process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
  #       /
  #       on(instance) group_left()
  #       tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
  #     ) > 0.8
  #   for: 10m
  #   labels:
  #     tier: next-gen
  #     severity: warning
  #     component: tikv
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
  #     value: '{{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?tab=t.0#heading=h.4wsh75ki5tyw
  #     summary: Max memory utilization across TiKV nodes exceeded 80% for 10 minutes
  #     message: |
  #       Maximum TiKV node memory utilization in cluster {{ $labels.tidb_cluster_id }} has exceeded 80% for 10 minutes. High memory usage may affect TiKV performance and stability. Consider increasing the node memory size or adding more TiKV nodes to distribute the memory load.
  # - alert: MaxTiKVCPUUsageTooHighInternal
  #   expr: |-
  #     max(
  #       sum by (instance) (
  #         rate(process_cpu_seconds_total{component="tikv", job=~".*tikv.*"}[2m])
  #       )
  #       /
  #       on(instance)
  #       tikv_server_cpu_cores_quota{component="tikv", job=~".*tikv.*"}
  #     ) by (tidb_cluster_id) > 0.8
  #   for: 10m
  #   labels:
  #     tier: next-gen
  #     severity: warning
  #     component: tikv
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
  #     value: '{{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?tab=t.0#heading=h.djq2y3lhinpm
  #     summary: Max CPU utilization across TiKV nodes exceeded 80% for 10 minutes
  #     message: |
  #       Maximum TiKV node CPU utilization in cluster {{ $labels.tidb_cluster_id }} has exceeded 80% for 10 minutes. High CPU usage may affect TiKV performance and response times. Consider increasing the node CPU capacity or adding more TiKV nodes to distribute the computational load.
  - alert: TiKVInstanceLowSpace
    expr: |-
      (
        sum(tikv_store_size_bytes{type="available"}) by (tidb_cluster_id, instance)
        / sum(tikv_store_size_bytes{type="capacity"}) by (tidb_cluster_id, instance) * 100 < 30
      )
      and
      (
        sum(tikv_store_size_bytes{type="available"}) by (tidb_cluster_id, instance) < 800 * 1024 * 1024 * 1024
      )
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has low disk space. Available space is less than 30% ({{ $value }}) and remaining capacity is less than 800GB. Please consider expanding storage capacity or cleaning up data.'
      value: '{{ $value }}'
      summary: "TiKV store space available less than 30% and remaining less than 800GB"
  - alert: TiKVInstanceVeryLowSpace
    expr: sum(tikv_store_size_bytes{type="available"}) by (tidb_cluster_id, instance) / sum(tikv_store_size_bytes{type="capacity"}) by (tidb_cluster_id, instance) * 100 < 10
    for: 5m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has critically low disk space. Available space is less than 10% ({{ $value }}). Immediate action required to prevent data write failures.'
      value: '{{ $value }}'
      summary: "TiKV store space used more than 90%"

  - alert: TiKVSuspectedDeadlock
    expr: |-
      (
        sum(rate(tikv_thread_cpu_seconds_total{name=~"(raftstore|rs|rfstore)_.*"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type) == 0
      )
      and
      (
        sum(rate(tikv_thread_cpu_seconds_total{name=~"grpc.*"}[1m])) by (tidb_cluster_id, instance, tenant, provider_type) == 0
      )
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV suspected deadlock detected (cluster: {{ $labels.tidb_cluster_id }}, instance {{ $labels.instance }})"
      description: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} is suspected of being deadlocked. Both rfstore and gRPC threads are reporting 0% CPU usage for over 10 minutes. This indicates the TiKV process may be stuck and unable to process requests. Immediate investigation and potential restart of the TiKV instance is recommended. value: {{ $value }}'