groups:
- name: tiflash
  rules:
  #- alert: TiFlashServerDowntimeIsTooLong
  #  expr: up{component=".*tiflash"} == 0 and max_over_time(up{component=".*tiflash", job=~".*tiflash.*"}[100m] offset 10m) > 0
  #  for: 10m
  #  labels:
  #    tier: next-gen
  #    severity: major
  #    expr: up{component=".*tiflash"} == 0
  #    component: tiflash
  #  annotations:
  #    description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #    runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
  #    value: '{{ $value }}'
  #    summary: TiFlash server downtime is more than 5 min
  - alert: TiFlashServerFrequentlyRestart
    expr: |-
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
          resets((-floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}))[1h:15s])
        and
          max_over_time(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiflash
      oncall_team: tiflash
    annotations:
      description: 'TiFlash server {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has restarted {{ $value }} times in the past hour. Frequent TiFlash restarts can affect analytical query processing and columnar storage operations. Please investigate the root cause and ensure TiFlash server stability. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash server is down for at least 3 times in a hour
  - alert: TiFlashClusterFrequentlyRestart
    expr: |-
      (
        sum by (tidb_cluster_id, tenant, provider_type) (
            resets((-floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}))[1h:15s])
          and
            max_over_time(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}))[1h:15s])
            ) by (tidb_cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (tidb_cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tiflash
      oncall_team: tiflash
    annotations:
      description: 'Multiple TiFlash servers in cluster {{ $labels.tidb_cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting TiFlash stability. Please investigate the cluster infrastructure and TiFlash configuration to prevent further restarts. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash servers in cluster are down for at least 3 times in a hour
  # warning level always missing
  # - alert: TiFlashServerRestarted
  #   expr: sum(resets((-floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}))[1h:15s])) by (cluster_id, instance, tenant, provider_type) >= 1 and sum(max_over_time(up{job=~".*tiflash.*"}[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
  #   for: 10m
  #   labels:
  #     tier: next-gen
  #     severity: warning
  #     component: tiflash
  #   annotations:
  #     description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #     runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
  #     value: '{{ $value }}'
  #     summary: TiFlash server is down for at least 1 times in a hour
  #     message: |
  #       TiFlash server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has restarted {{ $value }} time(s) in the past hour. Please monitor the TiFlash server for stability and investigate if restarts become frequent.
  - alert: TiFlashInodeUsageMoreThan90%
    expr: |
      kubelet_volume_stats_inodes_used{persistentvolumeclaim=~".+-db-[a-z0-9]+-(write|compute)-tiflash-.+"}/kubelet_volume_stats_inodes{persistentvolumeclaim=~".+-db-[a-z0-9]+-(write|compute)-tiflash-.+"} > 0.9
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: next-gen
      component: tiflash
      oncall_team: tiflash
    annotations:
      description: 'TiFlash inode usage for PVC {{ $labels.persistentvolumeclaim }} in namespace {{ $labels.namespace }} has exceeded 90% ({{ $value }}). High inode usage may prevent new file creation and affect TiFlash operations. Please clean up unnecessary files or expand storage capacity to resolve the inode shortage.'
      value: '{{ $value }}'
      runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
      summary: "TiFlash inode usage more than 90%"
  - alert: TiFlashRaftstoreCPUDropToZero
    # Add condition to avoid false positives when TiFlash store size is less than 1 MB, which may indicate that the store is not active or has just started.
    expr: |-
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
        rate(tiflash_proxy_thread_cpu_seconds_total{name=~"raftstore_.*", instance=~".*write-tiflash.*"}[2m])
      ) == 0
      and on (tidb_cluster_id, instance, tenant, provider_type)
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
        tiflash_system_current_metric_StoreSizeUsed{instance=~".*write-tiflash.*"}
      ) > 1024 * 1024
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tiflash
      oncall_team: tiflash
    annotations:
      summary: The Raft store CPU of a single TiFlash drops to 0 and persists for more than 10 minutes.
      description: 'The Raft store CPU of a single TiFlash drops to 0 and persists for more than 10 minutes in cluster {{ $labels.tidb_cluster_id }} on instance {{ $labels.instance }}. This may indicate that the TiFlash Raft store hangs, which will affect the data replication of TiFlash. Please check the TiFlash logs and take mitigation measures.'