groups:
- name: tiflash
  rules:
  - alert: TiFlashServerIsDown
    expr: up{component="tiflash"} == 0
    for: 10m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: critical
      response_class: page
      expr: up{component="tiflash"} == 0
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.ig9wflpfgm34
      value: '{{ $value }}'
      summary: TiFlash server is down
  - alert: TiFlashLowSpace
    expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id, instance, tenant,provider_type) /  sum(tiflash_system_current_metric_StoreSizeCapacity) by (cluster_id, instance, tenant,provider_type) * 100 < 10
    for: 5m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id, instance, tenant,provider_type) /  sum(tiflash_system_current_metric_StoreSizeCapacity) by (cluster_id, instance, tenant,provider_type) * 100 < 10
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
      value: '{{ $value }}'
      summary: "TiFlash store space used more than 90%"
  - alert: TiFlashVeryLowSpace
    expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id, instance, tenant,provider_type) /  sum(tiflash_system_current_metric_StoreSizeCapacity) by (cluster_id, instance, tenant,provider_type) * 100 < 5
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(tiflash_system_current_metric_StoreSizeAvailable) by (cluster_id, instance, tenant,provider_type) /  sum(tiflash_system_current_metric_StoreSizeCapacity) by (cluster_id, instance, tenant,provider_type) * 100 < 5
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.skuuhxdpaymf
      value: '{{ $value }}'
      summary: "TiFlash store space used more than 95%"
  - alert: TiflashRestartsFrequently
    expr: sum(increase(kube_pod_container_status_restarts_total{container="tiflash"}[30m]))by(container,tenant,namespace,cluster_id)>5
    for: 10m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(increase(kube_pod_container_status_restarts_total{container="tiflash"}[30m]))by(container,tenant,namespace,cluster_id)>5
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      summary: Tiflash container restart frequently
      message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times in 30 minutes.
      value: '{{ $value }}'
  - alert: TiFlashInodeUsageMoreThan90%
    expr: |
      kubelet_volume_stats_inodes_used{persistentvolumeclaim=~".*tiflash-.*"}/kubelet_volume_stats_inodes{persistentvolumeclaim=~".*tiflash-.*"} > 0.9
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      description: 'inode usage more than 90%, region: {{$labels.region}}, cluster: {{ $labels.tidb_cluster }}, instance: {{ $labels.instance }}, value: {{ $value }}. cluster_id: {{ $labels.cluster_id }}'
      value: '{{ $value }}'
      runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
      summary: 'TiFlash inode usage more than 90%'
  - alert: NoAvailableTiFlashAutoscaler
    expr: |
      kube_statefulset_status_replicas_available{namespace="tiflash-autoscale", statefulset="autoscale"} == 0
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      expr: kube_statefulset_status_replicas_available{namespace="tiflash-autoscale", statefulset="autoscale"} == 0
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: No available tiflash autoscaler
  - alert: NoAvailableTiFlashReadNode
    expr: |
      kube_pod_status_ready{namespace="tiflash-autoscale", condition="true", pod=~"readnode-.*"} == 0
    for: 5m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      expr: kube_pod_status_ready{namespace="tiflash-autoscale", condition="true", pod=~"readnode-.*"} == 0
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: No available tiflash readnode
  - alert: TiFlashMemoryWarn
    expr: sum(tiflash_proxy_process_resident_memory_bytes) by (cluster_id, instance, tenant, provider_type) /  sum(tiflash_system_current_metric_MemoryCapacity) by (cluster_id, instance, tenant, provider_type) * 100 > 80
    for: 10m
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(tiflash_proxy_process_resident_memory_bytes) by (cluster_id, instance, tenant,provider_type) /  sum(tiflash_system_current_metric_MemoryCapacity) by (cluster_id, instance, tenant, provider_type) * 100 > 80
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      summary: "TiFlash memory used more than 80% for 10min"
  - alert: TiFlashMemoryCrit
    expr: sum(tiflash_proxy_process_resident_memory_bytes) by (cluster_id, instance, tenant, provider_type) /  sum(tiflash_system_current_metric_MemoryCapacity) by (cluster_id, instance, tenant, provider_type) * 100 > 90
    for: 5m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      expr: sum(tiflash_proxy_process_resident_memory_bytes) by (cluster_id, instance, tenant, provider_type) /  sum(tiflash_system_current_metric_MemoryCapacity) by (cluster_id, instance, tenant, provider_type) * 100 > 90
      component: tiflash
      oncall_team: tikv_unified
    annotations:
      message: 'cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      value: '{{ $value }}'
      summary: "TiFlash memory used more than 90% for 5min"