groups:
- name: tikv/tikv
  rules:
  - alert: TiKVServerIsDown
    expr: up{component="tikv"} == 0
    for: 10m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      expr: up{component="tikv"} == 0
      component: tikv
      oncall_team: tikv_unified
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server is down
  - alert: TiKVServerIsDownTooLong
    expr: up{component="tikv"} == 0
    for: 24h
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      expr: up{component="tikv"} == 0
      component: tikv
      oncall_team: tikv_unified
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server is down for 24h
  - alert: TiKVServerFrequentlyDown
    expr: sum(changes(process_start_time_seconds{component="tikv"}[1h])) by (instance, region) >=4
    for: 10m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      expr: sum(changes(process_start_time_seconds{component="tikv"}[1h])) by (instance, region) >=4
      component: tikv
      oncall_team: tikv_unified
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.t0ws8abzblbu
      value: '{{ $value }}'
      summary: TiKV server is down for at least 4 times in a hour
  - alert: TiKVRaftAppendLogSlow
    expr: histogram_quantile(0.99, sum(rate(tikv_raftstore_store_write_raftdb_duration_seconds_bucket[1m])) by (le, instance, region)) > 30
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's raft append log P99 duration in seconds is too slow (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's raft append log P99 duration is too slow\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVLowSpace
    expr: (sum(tikv_store_size_bytes{type="available"}) by (instance, region)  / sum(tikv_store_size_bytes{type="capacity"}) by (instance, region)  * 100 < 20) and (sum(tikv_store_size_bytes{type="available"}) by (instance, region) < 500 * 1024 * 1024 * 1024)
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr:  (sum(tikv_store_size_bytes{type="available"}) by (instance, region)  / sum(tikv_store_size_bytes{type="capacity"}) by (instance, region)  * 100 < 20) and (sum(tikv_store_size_bytes{type="available"}) by (instance, region) < 500 * 1024 * 1024 * 1024)
      component: tikv
      oncall_team: tikv_unified
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.3tz23916jlht
      value: '{{ $value }}'
      summary: "TiKV store space used more than 80%"
  - alert: TiKVVeryLowSpace
    expr: (sum(tikv_store_size_bytes{type="available"}) by (instance, region)  / sum(tikv_store_size_bytes{type="capacity"}) by (instance, region)  * 100 < 10) and (sum(tikv_store_size_bytes{type="available"}) by (instance, region) < 500 * 1024 * 1024 * 1024)
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      expr: (sum(tikv_store_size_bytes{type="available"}) by (instance, region)  / sum(tikv_store_size_bytes{type="capacity"}) by (instance, region)  * 100 < 10) and (sum(tikv_store_size_bytes{type="available"}) by (instance, region) < 500 * 1024 * 1024 * 1024)
      component: tikv
      oncall_team: tikv_unified
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.w4zl0kl6ny53
      value: '{{ $value }}'
      summary: "TiKV store space used more than 90%"
  - alert: TiKVRaftstoreCpuHighLoad
    expr: max(rate(tikv_thread_cpu_seconds_total{namespace="tidb-serverless", container="tikv", name="rfstore_0"}[1m])) by (region, instance) > 0.85
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      expr: max(rate(tikv_thread_cpu_seconds_total{namespace="tidb-serverless", container="tikv", name="rfstore_0"}[1m])) by (region, instance) > 0.85
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "raftstore high CPU load"
      message: "TiKV node's raftstore thread is under a high load\n region: {{ $labels.region }}, instance: {{ $labels.instance }}, VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ynohpbfagetz
  - alert: TiKVRaftstoreCpuFull
    expr: max(rate(tikv_thread_cpu_seconds_total{namespace="tidb-serverless", container="tikv", name="rfstore_0"}[1m])) by (region, instance) > 0.95
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      expr: max(rate(tikv_thread_cpu_seconds_total{namespace="tidb-serverless", container="tikv", name="rfstore_0"}[1m])) by (region, instance) > 0.95
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "raftstore CPU is Full"
      message: "TiKV node's raftstore thread is full\n region: {{ $labels.region }}, instance: {{ $labels.instance }}, VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?heading=h.ynohpbfagetz
  - alert: TiKVApplyWorkerPoolCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+"}[1m])) by (instance, region) / count(tikv_thread_cpu_seconds_total{name=~"apply_[0-9]+"}) by (instance, region)  > 0.9
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Apply worker thread pool high CPU load (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's apply worker thread pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.7k1o575l3zea
  - alert: TiKVSchedulerWorkerCpuHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"sched_.*"}[1m])) by (instance, region) / count(tikv_thread_cpu_seconds_total{name=~"sched_.*"}) by (instance, region)  > 0.9
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Scheduler worker high CPU load (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's scheduler worker thread pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.qptrnffkw5h
  - alert: TiKVGrpcPollHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"grpc.*"}[1m])) by (instance, region)  / count(tikv_thread_cpu_seconds_total{name=~"grpc.*"}) by (instance, region)   > 0.8
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "gRPC high CPU load (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's gRPC polling thread pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.30ra2mgpp1i4
  - alert: TiKVUnifiedReadPoolHighLoad
    expr: sum(rate(tikv_thread_cpu_seconds_total{name=~"unified_read_po*"}[1m])) by (instance, region) / count(tikv_thread_cpu_seconds_total{name=~"unified_read_po*"}) by (instance, region)  > 0.9
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "Unified read pool high CPU load (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's unified read pool is under a high load\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.y1jc473vb50t
  - alert: TiKVTooManyFailedGRPCRequests
    expr: rate(tikv_grpc_msg_fail_total{type!="kv_gc"}[1m]) > 100
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node has too many grpc requests failed (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's has too many grpc requests failed\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
  
  - alert: TiKVReportTooManyFailureMsg
    expr:  sum(rate(tikv_server_report_failure_msg_total{type="unreachable"}[1m])) BY (instance) > 100
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node reported too many unreachable failure messages (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node reported too many unreachable failure messages\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVChannelFull
    expr: sum(rate(tikv_channel_full_total[1m])) BY (type, instance) > 0
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's gRPC channel is full (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's gRPC channel is full\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o
  
  - alert: TiKVRaftLogLag
    expr: histogram_quantile(1, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, instance, region))  > 5000
    for: 1m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV node's Raft log lap is more than 5000 (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV node's Raft log lap is more than 5000\n  VALUE = {{ $value }}"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVTooManyRegions
    expr: |
      sum(tikv_raftstore_region_count{type="region"}) by (instance, region)
      > on(instance) group_left() tikv:serverless_instance_region_limit
    for: 6h
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV instance region count exceeds CPU-tier limit (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "region: {{ $labels.region }}, instance: {{ $labels.instance }}, current region count: {{ $value }}"

  - alert: TiKVBackupFailure
    expr: changes(raft_engine_backup_counter{status=~".*_fail"}[10m]) > 0
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV auto backup fails (region: {{ $labels.region }}, instance {{ $labels.instance }})"
      message: "TiKV auto backup fails \n"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?n=DBaaS_TiDB_Alert_Runbook#heading=h.pj8pl2jqoyh6
  - alert: TiKVNoSuccessfulBackup
    expr: sum(increase(native_br_backup_success[1h])) == 0
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV auto backup has no successful run for 1 hour"
      message: "TiKV auto backup has no successful run in the last hour."
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?n=DBaaS_TiDB_Alert_Runbook#heading=h.pj8pl2jqoyh6
  - alert: TiKVInodeUsageMoreThan90%
    expr: |
      kubelet_volume_stats_inodes_used{persistentvolumeclaim=~".*tikv-.*"}/kubelet_volume_stats_inodes{persistentvolumeclaim=~".*tikv-.*"} > 0.9
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: tikv
      oncall_team: tikv_unified
    annotations:
      description: 'inode usage more than 90%, region: {{$labels.region}}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
      summary: "TiKV inode usage more than 90%"

  - alert: TiKVRegionBlacklisted
    expr: increase(tikv_raftstore_blacklist_region_total[10m]) > 0
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV blacklisted region"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, new blacklisted shards: {{ $value }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnPijSXMwCtwlIn36l4V2SYe

  - alert: TiKVBlacklistRegionsNonZero
    expr: tikv_raftstore_blacklist_region_total > 0
    for: 24h
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV has blacklisted regions"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, blacklisted shards: {{ $value }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnPijSXMwCtwlIn36l4V2SYe

  - alert: TiKVRaftEngineDfsWorkerUnhealthy
    expr: raft_engine_dfs_worker_healthy == 0
    for: 1m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV raft engine dfs worker not healthy"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, raft engine dfs worker unhealthy"
      runbook_url: https://pingcap-cn.feishu.cn/wiki/JHdMwA40PiNdqmkUg9QcLVgDn2b

  - alert: TiKVIAStorageClassTransitionFailed
    expr: sum(increase(kv_engine_ia_transition_region_finished_total{result!="success"}[10m])) by (instance, region, result) > 0
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV IA storage-class transition failed"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, result: {{ $labels.result }}, value: {{ $value }}"

  - alert: TiKVIAStorageClassTransitionStuck
    expr: |
      (
        sum(kv_engine_ia_transition_region_pending) by (instance, region)
        + sum(kv_engine_ia_transition_region_inflight) by (instance, region)
      ) > 0
      unless on(instance, region)
      (
        sum(increase(kv_engine_ia_transition_region_finished_total[30m])) by (instance, region) > 0
      )
    for: 30m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: major
      response_class: ticket
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV IA storage-class transition may be stuck"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, value: {{ $value }}"

  - alert: TiKVIARemoteReadLatencyHigh
    expr: |
      histogram_quantile(0.999,
        sum(rate(kv_engine_ia_remote_read_segment_duration_seconds_bucket[5m])) by (le, instance, region)
      ) > 1
      and on(instance, region)
      sum(rate(kv_engine_ia_remote_read_segment_duration_seconds_count[5m])) by (instance, region) > 1
    for: 15m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV IA remote read p999 latency is high"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, p999 latency: {{ $value }}s"

  - alert: TiKVDuplicatedChangeSet
    expr: increase(kv_engine_duplicated_changeset_counter[10m]) > 0
    for: 30m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV duplicated changeset"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, duplicated changeset count: {{ $value }}"

  - alert: TiKVCompactFailed
    expr: increase(kv_engine_compaction_result_counter{result="failure"}[10m]) > 0
    for: 30m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV compaction failure"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, compaction failure count: {{ $value }}"

  - alert: TiKVCompactSchemaOutOfDate
    expr: increase(kv_engine_compaction_result_counter{result="schema_out_of_date"}[10m]) > 0
    for: 30m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV compaction with stale schema"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, compaction count: {{ $value }}"

  - alert: TiKVCompactFallbackFailed
    expr: increase(kv_engine_compaction_result_counter{result="local_compaction_disabled"}[10m]) > 0
    for: 30m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV compaction fallback failed"
      message: "region: {{$labels.region}}, instance: {{ $labels.instance }}, compaction count: {{ $value }}"

  - alert: TiKVSchedulerTooManyPendingTasks
    expr: avg_over_time(tikv_scheduler_context_total[5m]) > 2000
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV scheduler has too many pending tasks"
      message: "TiKV scheduler has too many pending tasks (region: {{ $labels.region }}, instance: {{ $labels.instance }}, value: {{ $value }})"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.th5j33flx0o

  - alert: TiKVTxnFileSplitDeniedRegions
    expr: sum(increase(tikv_rfstore_txn_file_split_denied_regions_total[1m])) by (region) > 16
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV txn file split denied regions"
      message: "TiKV txn file split denied regions (region: {{ $labels.region }}, value: {{ $value }})"

  - alert: TiKVTxnFilePrewriteRejected
    expr: sum(increase(tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total[1m])) by (region) > 16
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: "TiKV txn file prewrite rejected"
      message: "TiKV txn file prewrite rejected (region: {{ $labels.region }}, value: {{ $value }})"