groups:
- name: tiflash
  rules:
  #- alert: TiFlashServerDowntimeIsTooLong
  #  expr: up{component="tiflash"} == 0 and max_over_time(up{component="tiflash", job=~".*tiflash"}[100m] offset 10m) > 0
  #  for: 10m
  #  labels:
  #    tier: dedicated
  #    severity: major
  #    expr: up{component="tiflash"} == 0
  #    component: tiflash
  #  annotations:
  #    description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
  #    runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
  #    value: '{{ $value }}'
  #    summary: TiFlash server downtime is more than 5 min
  - alert: TiFlashServerDowntimeIsTooLong
    expr: up{component="tiflash"} == 0
    for: 10m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      expr: up{component="tiflash"} == 0
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash server downtime is more than 10 min
  - alert: TiFlashServerFrequentlyRestart
    expr: sum(changes(floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"})[1h:15s])) by (cluster_id, instance, tenant, provider_type) >=3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      expr: sum(changes(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}[1h])) by (cluster_id, instance, tenant, provider_type) >=3
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash server is down for at least 3 times in a hour
  - alert: TiFlashClusterFrequentlyRestart
    expr: |-
      sum by (cluster_id, tenant, provider_type) (
          resets((-floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}))[1h:15s])
        and
          max_over_time(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: dedicated
      severity: critical
      response_class: page
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash servers in cluster are down for at least 3 times in a hour
  - alert: TiFlashServerRestarted
    expr: sum(changes(floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"})[1h:15s])) by (cluster_id, instance, tenant, provider_type) >=1 and sum(max_over_time(up{job=~".*tiflash"}[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
    for: 10m
    labels:
      tier: dedicated
      severity: warning
      expr: sum(changes(floor(tiflash_proxy_process_start_time_seconds{job=~".*tiflash.*"})[1h:15s])) by (cluster_id, instance, tenant, provider_type) >=1 and sum(max_over_time(up{job=~".*tiflash"}[100m] offset 60m)) by (cluster_id, instance, tenant, provider_type) > 0
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash server is down for at least 1 times in a hour
  # Dynamic threshold: (1 - low_space_ratio) * 100 + 10
  # default low_space_ratio=0.8 → available < 30%, custom 0.9 → available < 20%
  - alert: TiFlashInstanceLowSpace
    expr: |
      (
        sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id, instance, tenant, provider_type)
        / sum(tiflash_system_current_metric_StoreSizeCapacity) by (cluster_id, instance, tenant, provider_type) * 100
      )
      < on(cluster_id) group_left()
      (
        (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 100 + 10
      )
    for: 5m
    labels:
      severity: warning
      tier: dedicated
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://pingcap.feishu.cn/wiki/MjlmwjvnBiL5fBkZLpYcXGDOnIh
      value: '{{ $value }}'
      summary: "TiFlash store space approaching low-space-ratio boundary"
  # Dynamic threshold: (1 - low_space_ratio) * 100 + 2
  # default low_space_ratio=0.8 → available < 22%, custom 0.9 → available < 12%
  - alert: TiFlashInstanceVeryLowSpace
    expr: |
      (
        sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id, instance, tenant, provider_type)
        / sum(tiflash_system_current_metric_StoreSizeCapacity) by (cluster_id, instance, tenant, provider_type) * 100
      )
      < on(cluster_id) group_left()
      (
        (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 100 + 2
      )
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: dedicated
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://pingcap.feishu.cn/wiki/MjlmwjvnBiL5fBkZLpYcXGDOnIh
      value: '{{ $value }}'
      summary: "TiFlash store space very close to low-space-ratio boundary"
  - alert: TiFlashInodeUsageMoreThan90%
    expr: |
      kubelet_volume_stats_inodes_used{component="tiflash"}/kubelet_volume_stats_inodes{component="tiflash"} > 0.9
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: dedicated
      component: tiflash
      oncall_team: resilience
    annotations:
      description: 'inode usage more than 90%, cluster: {{ $labels.tidb_cluster }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
      summary: "TiFlash inode usage more than 90%"