groups:
- name: next-gen-tiflash-kernel-tcocp.rules
  rules:
  - alert: TiFlashWriteLowSpace70NextGen
    expr: |-
      sum(tiflash_system_current_metric_StoreSizeAvailable{pod=~".*-write-tiflash.*"}) by (cluster_id, instance, tenant, provider_type)
      / sum(tiflash_system_current_metric_StoreSizeCapacity{pod=~".*-write-tiflash.*"}) by (cluster_id, instance, tenant, provider_type) * 100
      < 30
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-2864
      sop_url: https://pingcap.feishu.cn/wiki/MjlmwjvnBiL5fBkZLpYcXGDOnIh
    annotations:
      summary: TiFlash-write store space used more than 70%
      description: 'TiFlash-write store space used more than 70%. source_tcoc: TCOC-2864 sop_url: https://pingcap.feishu.cn/wiki/MjlmwjvnBiL5fBkZLpYcXGDOnIh. cluster: {{ $labels.tidb_cluster_id }}, value: {{ $value }}'
      value: '{{ $value }}'
  - alert: TiFlashWriteLowSpace78NextGen
    expr: |-
      sum(tiflash_system_current_metric_StoreSizeAvailable{pod=~".*-write-tiflash.*"}) by (cluster_id, instance, tenant, provider_type)
      / sum(tiflash_system_current_metric_StoreSizeCapacity{pod=~".*-write-tiflash.*"}) by (cluster_id, instance, tenant, provider_type) * 100
      < 22
    for: 5m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-2864
      sop_url: https://pingcap.feishu.cn/wiki/MjlmwjvnBiL5fBkZLpYcXGDOnIh
    annotations:
      summary: TiFlash-write node store space used more than 78%
      description: 'TiFlash-write store space used more than 78%. source_tcoc: TCOC-2864 sop_url: https://pingcap.feishu.cn/wiki/MjlmwjvnBiL5fBkZLpYcXGDOnIh. cluster: {{ $labels.tidb_cluster_id }}, value: {{ $value }}'
      value: '{{ $value }}'
  - alert: TiFlashComputeLowSpace95NextGen
    expr: |-
      sum(tiflash_system_current_metric_StoreSizeAvailable{pod=~".*-compute-tiflash.*"}) by (cluster_id, instance, tenant, provider_type)
      / sum(tiflash_system_current_metric_StoreSizeCapacity{pod=~".*-compute-tiflash.*"}) by (cluster_id, instance, tenant, provider_type) * 100
      < 5
    for: 5m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-2864
    annotations:
      summary: TiFlash-compute store space used more than 95%
      description: 'TiFlash-compute store space used more than 95%. source_tcoc: TCOC-2864. cluster: {{ $labels.tidb_cluster_id }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
      value: '{{ $value }}'
  - alert: TiFlashRestartFrequentlyNextGen
    expr: sum(increase(kube_pod_container_status_restarts_total{container="tiflash"}[30m])) > 5
    for: 10m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-3758
    annotations:
      summary: Tiflash container restart frequently
      message: |
        Container {{ $labels.namespace }}/{{ $labels.container }} pod {{ $labels.pod }} is restarting {{ printf "%.2f" $value }} times in 30 minutes. source_tcoc: TCOC-3758
  - alert: TiFlashWriteThroughputTooHighNextGen
    expr: sum by (instance) (rate(tiflash_storage_throughput_bytes{job=~".*tiflash.*",type="write"}[2m])) > 100 * 1024 * 1024
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-3865
    annotations:
      summary: The write throughput of one TiFlash node has exceeded 100 MiB/s and lasted for 10 minutes.
      description: 'The write throughput of one TiFlash node has exceeded 100 MiB/s and lasted for 10 minutes. source_tcoc: TCOC-3865 dash_board: TiFlash-Summary -> Storage Write Stall -> Write Throughput By Instance -> "write-xxx". One TiFlash node''s write throughput exceeds 100 MiB/s and lasts for 10 minutes in cluster {{ $labels.tidb_cluster_id }}. This may indicate a sudden change in the business or a sudden increase in write traffic caused by DDL reorg. Please confirm whether the TiFlash raft-log synchronization delay has increased and take mitigation measures.'
      value: '{{ $value }}'
  - alert: TiFlashHighRaftWaitIndexNextGen
    expr: histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket[2m])) by (le)) > 60
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-3968
    annotations:
      summary: The Raft index of a single TiFlash exceeds 60s and persists for more than 5 minute.
      message: |
        The Raft index of a single TiFlash exceeds 60s and persists for more than 5 minute. TiDB Cluster: {{ $labels.tidb_cluster_id }} source_tcoc: TCOC-3968 dashboard: TiFlash-Summary -> Raft -> Raft Wait Index Duration
  - alert: TiFlashProxyEntryCacheHighMemoryNextGen
    expr: |-
      (
            tiflash_proxy_tikv_server_mem_trace_sum{name="raftstore-entry_cache"}
          / on (instance)
            tiflash_proxy_process_resident_memory_bytes
        )
      >
        0.2
    for: 0s
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-3940
    annotations:
      summary: The entry cache usage of TiFlash Proxy exceeds 20% of the memory.
      message: |
        The entry cache usage of TiFlash Proxy exceeds 20% of the memory. source_tcoc: TCOC-3940 dashboard: TiFlash-Proxy-Detail -> Cluster -> Raft Entry Cache
      value: '{{ $value }}'
  - alert: TiFlashOOMRelatedPodRestartsNextGen
    expr: |-
      sum by (namespace, pod) (
        increase(kube_pod_container_status_restarts_total{container="tiflash",pod=~".*tiflash-.*"}[15m]) > 0
      )
      * on (namespace, pod) group_left ()
        max by (namespace, pod) (
          kube_pod_container_status_last_terminated_reason{container="tiflash",pod=~".*tiflash-.*",reason="OOMKilled"} > 0
        )
    for: 0s
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiflash
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tiflash
      oncall_team: tiflash
      source_tcoc: TCOC-4157
    annotations:
      message: |
        TiFlash Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled. source_tcoc: TCOC-4157