groups:
- name: tikv
  rules:
  #- alert: TiKVServerDowntimeIsTooLong
  #  expr: up{component="tikv", job=~".*tikv"} == 0
  #  for: 5m
  #  labels:
  #    tier: dedicated
  #    severity: major
  #    expr: up{component="tikv", job=~".*tikv"} == 0
  #    component: tikv
  #  annotations:
  #    description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #    runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
  #    value: '{{ $value }}'
  #    summary: TiKV server downtime is more than 5 min
  - alert: TiKVServerDowntimeIsTooLong
    expr: up{component="tikv", job=~".*tikv"} == 0
    for: 10m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      expr: up{component="tikv", job=~".*tikv"} == 0
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server downtime is more than 10 min
  - alert: TiKVServerFrequentlyRestart
    expr: sum(changes(floor(process_start_time_seconds{job=~".*tikv"})[1h:15s])) by (cluster_id, instance, tenant, provider_type) >=3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      expr: sum(changes(process_start_time_seconds{job=~".*tikv"}[1h])) by (cluster_id, instance, tenant, provider_type) >=3
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server is down for at least 3 times in a hour
  - alert: TiKVClusterFrequentlyRestart
    expr: |-
      (
        sum by (cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*tikv"}))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*tikv"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*tikv"}))[1h:15s])
            ) by (cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV instances for a cluster is down for at least 3 times in a hour
  - alert: TiKVServerRestarted
    expr: sum(changes(floor(process_start_time_seconds{job=~".*tikv"})[1h:15s])) by (cluster_id, instance, tenant, provider_type) >=1 and sum(max_over_time(up{job=~".*tikv"}[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
    for: 10m
    labels:
      tier: dedicated
      severity: warning
      expr: sum(changes(floor(process_start_time_seconds{job=~".*tikv"})[1h:15s])) by (cluster_id, instance, tenant, provider_type) >=1 and sum(max_over_time(up{job=~".*tikv"}[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server is down for at least 1 times in a hour
  # - alert: TiKVServerDownMoreThan1AZ
  #   expr: |
  #       count(count(
  #         label_replace((kube_pod_info{pod=~"db-tikv.*"} * on(node) group_left(provider_id) kube_node_info),"az_id",  "$1", "provider_id", "(.*/)(\\w+.*)")
  #         * on(pod,namespace) group_right(az_id)
  #         label_replace(label_replace(up{component="tikv"},"pod",  "$0", "instance", ".*"),"namespace",  "$0", "cluster_id", ".*") == 0
  #       ) by (cluster_id, az_id)) by (cluster_id) > 1
  #   for: 1m
  #   labels:
  #     severity: emergency
  #     component: tikv
  #     expr: |
  #         count(count(
  #           label_replace((kube_pod_info{pod=~"db-tikv.*"} * on(node) group_left(provider_id) kube_node_info),"az_id",  "$1", "provider_id", "(.*/)(\\w+.*)")
  #           * on(pod,namespace) group_right(az_id)
  #           label_replace(label_replace(up{component="tikv"},"pod",  "$0", "instance", ".*"),"namespace",  "$0", "cluster_id", ".*") == 0
  #         ) by (cluster_id, az_id)) by (cluster_id) > 1
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, down tikv servers which in different AZ, count: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ijox0linkuyj
  #     value: '{{ $value }}'
  #     summary: "TiKV server down count more than 1 and in different AZ"
  # - alert: TiKV_critical_error
  #   expr: sum(rate(tikv_critical_error_total[1m])) BY (cluster_id, instance, tenant, provider_type) > 0
  #   # without the for clause will become active on the first evaluation.
  #   labels:
  #     env: ENV_LABELS_ENV
  #     level: emergency
  #     expr: sum(rate(tikv_critical_error_total[1m])) BY (cluster_id, instance, tenant, provider_type) > 0
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?#heading=h.kq5ljh5ykmek
  #     value: '{{ $value }}'
  #     summary: "TiKV encounters critical errors"
  - alert: TiKVRaftstoreCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}) by (cluster_id, instance, tenant, provider_type)  > 0.8
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "Raftstore high CPU load (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's raftstore is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ynohpbfagetz
  - alert: TiKVApplyWorkerPoolCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+"}[1m])) by (cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+"}) by (cluster_id, instance, tenant, provider_type)  > 0.9
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "Apply worker thread pool high CPU load (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's apply worker thread pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.7k1o575l3zea
  - alert: TiKVSchedulerWorkerCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"sched_.*"}[1m])) by (cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"sched_.*"}) by (cluster_id, instance, tenant, provider_type)  > 0.9
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "Scheduler worker high CPU load (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's scheduler worker thread pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.qptrnffkw5h
  - alert: TiKVGrpcPollHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"grpc.*"}[1m])) by (cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"grpc.*"}) by (cluster_id, instance, tenant, provider_type)  > 0.8
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "gRPC high CPU load (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's gRPC polling thread pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.30ra2mgpp1i4
  - alert: TiKVUnifiedReadPoolHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"unified_read_po.*"}[1m])) by (cluster_id, instance, tenant, provider_type)  / count(tikv_thread_cpu_seconds_total{name=~"unified_read_po.*"}) by (cluster_id, instance, tenant, provider_type)  > 0.9
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "Unified read pool high CPU load (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's unified read pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.y1jc473vb50t

  - alert: TiKVTooManyRaftLogDroppedCount
    expr: sum(rate(tikv_server_raft_append_rejects{type!="kv_gc"}[1m])) by (cluster_id, instance, tenant, provider_type) > 100
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "Too many raft log dropped, likely due to excessive memory usage (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's dropped too many raft log under high memory usage\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftEntryCacheEvicts
    expr: rate(tikv_raft_entries_evict_bytes[1m]) > 5242880
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "Raft Entry Cache Evicts at high speed, likely due to excessive memory usage (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's Raft Entry Cache evicts at high speed, under high memory usage\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVTooManyFailedGRPCRequests
    expr: rate(tikv_grpc_msg_fail_total{type!="kv_gc"}[1m]) > 100
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "TiKV node has too many grpc requests failed (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's has too many grpc requests failed\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftAppendLogSlow
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le, cluster_id, instance, tenant, provider_type)) > 30
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "TiKV node's raft append log P99 duration in seconds is too slow (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's raft append log P99 duration is too slow\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftApplyLogDurationSecs
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le, cluster_id, instance, tenant, provider_type)) > 45
    for: 5m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "TiKV node's raft apply log P99 duration is too slow (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's raft apply log P99 duration in seconds is too slow\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
  # mute, confirmed by Lei Gao, 9.19, 2025.
  # - alert: TiKVReportTooManyFailureMsg
  #   expr:  sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[1m])) BY (cluster_id, instance, tenant, provider_type) > 100
  #   for: 5m
  #   labels:
  #     tier: dedicated
  #     severity: warning
  #     component: tikv
  #   annotations:
  #     summary: "TiKV node reported too many unreachable failure messages (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
  #     description: "TiKV node reported too many unreachable failure messages\n  VALUE = {{ $value }}"
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVChannelFull
    expr: sum(rate(tikv_channel_full_total[1m])) BY (type, cluster_id, instance, tenant, provider_type) > 0
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "TiKV node's gRPC channel is full (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's gRPC channel is full\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVRaftLogLag
    expr: histogram_quantile(1, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, cluster_id, instance, tenant, provider_type))  > 5000
    for: 1m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      summary: "TiKV node's Raft log lap is more than 5000 (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
      description: "TiKV node's Raft log lap is more than 5000\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVSlowScoreTooHigh
    expr: sum(tikv_raftstore_slow_score) by (cluster_id, instance, tenant, provider_type) >= 85
    for: 1m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      summary: "TiKV node reported a slow store (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"

  - alert: TiKVLargeLatencyNode
    expr: max(histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (cluster_id, instance, tenant, provider_type))) >= 5 * quantile(0.5, histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (cluster_id, instance, tenant, provider_type)))
    for: 5m
    labels:
      tier: dedicated
      severity: warning
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      summary: "TiKV node reported a slow TiKV node---its commit log duration is 5x more of the median value (cluster: {{ $labels.cluster_id }})"
  - alert: TiKVTooManyRegions
    expr: sum(tikv_raftstore_region_count{namespace=~".*1379661944636804070", type="region"}) by (cluster_id, instance, tenant, provider_type) > 160000
    for: 10m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, current region count: {{ $value }}'
      summary: "TiKV instance contains regions more than 160k (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
  - alert: TiKVTooManyRegions
    expr: sum(tikv_raftstore_region_count{namespace=~".*1379661944636804070", type="region"}) by (cluster_id, instance, tenant, provider_type) > 130000
    for: 10m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, current region count: {{ $value }}'
      summary: "TiKV instance contains regions more than 130k (cluster: {{ $labels.cluster_id }}, instance {{ $labels.instance }})"
  - alert: TiKVInodeUsageMoreThan90%
    expr: |
      kubelet_volume_stats_inodes_used{component="tikv"}/kubelet_volume_stats_inodes{component="tikv"} > 0.9
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: dedicated
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'inode usage more than 90%, cluster: {{ $labels.tidb_cluster }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      summary: "TiKV inode usage more than 90%"
  - alert: MaxTiKVMemoryUsageTooHighInternal
    expr: |-
      1 - min by (tidb_cluster_id) (
        avg by (instance, tidb_cluster_id) (
          (
            node_memory_MemFree_bytes{component="tikv"} +
            node_memory_Buffers_bytes{component="tikv"} +
            node_memory_Cached_bytes{component="tikv"}
          )
          /
          node_memory_MemTotal_bytes{component="tikv"}
        )
      ) > 0.7
    for: 10m
    labels:
      tier: dedicated
      severity: warning
      expr: |-
        1 - min by (tidb_cluster_id) (
          avg by (instance, tidb_cluster_id) (
            (
              node_memory_MemFree_bytes{component="tikv"} +
              node_memory_Buffers_bytes{component="tikv"} +
              node_memory_Cached_bytes{component="tikv"}
            )
            /
            node_memory_MemTotal_bytes{component="tikv"}
          )
        ) > 0.7
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      summary: Max memory utilization across TiKV nodes exceeded 70% for 10 minutes
  - alert: MaxTiKVCPUUsageTooHighInternal
    expr: max(avg(sum by (cpu, instance, tidb_cluster_id) (irate(node_cpu_seconds_total{component="tikv", mode!~"idle"}[2m]))) by (instance, tidb_cluster_id))by (tidb_cluster_id) > 0.8
    for: 10m
    labels:
      tier: dedicated
      severity: warning
      expr: max(avg(sum by (cpu, instance, tidb_cluster_id) (irate(node_cpu_seconds_total{component="tikv", mode!~"idle"}[2m]))) by (instance, tidb_cluster_id))by (tidb_cluster_id) > 0.8
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      summary: Max CPU utilization across TiKV nodes exceeded 80% for 10 minutes
  # Dynamic threshold: (1 - low_space_ratio) * 100
  # default low_space_ratio=0.8 → available < 20%, custom 0.9 → available < 10%
  - alert: TiKVInstanceLowSpace
    expr: |
      (
        tikv_store_size_bytes{type="available"}
        / tikv_store_size_bytes{type="capacity"} < 0.2
      )
      and
      (
        predict_linear(tikv_store_size_bytes{type="available"}[24h], 7 * 24 * 3600) < 0
      )
    for: 15m
    labels:
      tier: dedicated
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, available ratio: {{ $value }}'
      value: '{{ $value }}'
      summary: "TiKV store space is below 20% and predicted to fill up within 7 days"
  # Dynamic threshold: (1 - low_space_ratio) * 50
  # default low_space_ratio=0.8 → available < 10%, custom 0.9 → available < 5%
  - alert: TiKVInstanceVeryLowSpace
    expr: |
      (
        sum(tikv_store_size_bytes{type="available"}) by (cluster_id, instance)
        / sum(tikv_store_size_bytes{type="capacity"}) by (cluster_id, instance) * 100
      )
      < on(cluster_id) group_left()
      (
        (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 50
      )
    for: 5m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: tikv
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      summary: "TiKV store space very close to low-space-ratio boundary"