groups:
  - name: csngdataplane.rules
    rules:
      # Normal PV alert
      - alert: NextGenPersistentVolumeFillingUp
        annotations:
          message:
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is only {{ $value | humanizePercentage
            }} free.
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/KubePersistentVolumeFillingUp
        expr: |
          (
            kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*-db-.*", persistentvolumeclaim!~".*-compute-tiflash-.*", persistentvolumeclaim!~".*-coprocessor-.*", namespace=~"tidb.*"} /
            kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~".*-db-.*", persistentvolumeclaim!~".*-compute-tiflash-.*", persistentvolumeclaim!~".*-coprocessor-.*", namespace=~"tidb.*"} < 0.18
          )
          and
          kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*-db-.*", persistentvolumeclaim!~".*-compute-tiflash-.*", persistentvolumeclaim!~".*-coprocessor-.*", namespace=~"tidb.*"} < 536870912000
        for: 1m
        labels:
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          severity: critical
          response_class: page
          tier: next-gen
      # TiFlash Compute PV alert
      - alert: NextGenTiFlashCNPersistentVolumeFillingUp
        annotations:
          message:
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is only {{ $value | humanizePercentage
            }} free.
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/KubePersistentVolumeFillingUp
        expr: |
          kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*-compute-tiflash-.*", namespace=~"tidb.*"} /
          kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~".*-compute-tiflash-.*", namespace=~"tidb.*"} < 0.05
        for: 1m
        labels:
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          severity: critical
          response_class: page
          tier: next-gen
      # Coprocessor PV alert (90% full threshold)
      - alert: NextGenCoprocessorPersistentVolumeFillingUp
        annotations:
          message:
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim
            }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is only {{ $value | humanizePercentage
            }} free.
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/KubePersistentVolumeFillingUp
        expr: |
          (
            kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*-coprocessor-.*", namespace=~"tidb.*"} /
            kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~".*-coprocessor-.*", namespace=~"tidb.*"} < 0.1
          )
          and
          kubelet_volume_stats_available_bytes{persistentvolumeclaim=~".*-coprocessor-.*", namespace=~"tidb.*"} < 536870912000
        for: 1m
        labels:
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          severity: critical
          response_class: page
          tier: next-gen

      - alert: NextGenCPUUsageIsTooHigh
        annotations:
          message: CPU utilization of pod {{ $labels.pod }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is more than 95%.
          diagnosis: "Click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Ak6VpZD4z%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate%7B%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)%20by%20(pod)/sum(kube_pod_container_resource_limits%7Bresource%3D%5C%22cpu%5C%22,%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)%20by%20(pod))%20*100%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true,%22hide%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D"
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (namespace,pod)/
            sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (namespace,pod)
          ) *100 > 95
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          tier: next-gen
      - alert: NextGenAvgCPUUsageIsTooHigh
        annotations:
          message: Average CPU utilization across pods in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is more than 80%.
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (namespace,tidb_cluster_id)/
            sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (namespace,tidb_cluster_id)
          ) *100 > 80
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          tier: next-gen
      - alert: NextGenAvgCPUUsageIsCritical
        annotations:
          message: Average CPU utilization across pods in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is more than 90%.
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (namespace,tidb_cluster_id)/
            sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (namespace,tidb_cluster_id)
          ) *100 > 90
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          tier: next-gen

      - alert: NextGenMemUsageIsTooHigh
        annotations:
          message: Memory utilization of pod {{ $labels.pod }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is more than 90%.
          diagnosis: "Click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Ak6VpZD4z%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22(sum(container_memory_working_set_bytes%7Bcontainer!%3D%5C%22POD%5C%22,%20container!%3D%5C%22%5C%22,%20image!%3D%5C%22%5C%22,%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)by(pod)/sum(kube_pod_container_resource_limits%7Bresource%3D%5C%22memory%5C%22,%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)%20by%20(pod)%20*%20100)%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true,%22hide%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D"
        expr: |
          (sum(container_memory_working_set_bytes{namespace=~"tidb.*",container!="POD", container!="", image!="", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}
          - container_memory_total_active_file_bytes{namespace=~"tidb.*",container!="POD", container!="", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"})by(pod)/
          sum(kube_pod_container_resource_limits{resource="memory",namespace=~"tidb.*", pod!~".*-tikv-......|.*(tibr|tibrgc|restore|tiflash|o11y|cdc).*"}) by (pod)
          * 100) > 90
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          tier: next-gen

      # CPU/Memory Usage Low Alerts
      - alert: NextGenCPUUsageIsTooLow
        annotations:
          message: CPU utilization of pod {{ $labels.pod }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is less than 45% for recent 24 hours.
          summary: Pod CPU usage is idle, consider reducing resource allocation
        expr: |
          max_over_time(
            (
              sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"tidb[^-].*", pod!~".*-tikv-......|.*(tidb|worker|scheduling|pd|tso|cluster-agent|tibr|tibrgc|restore|tiflash|tiproxy|o11y|cdc).*"}) by (namespace,pod)/
              sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"tidb[^-].*", pod!~".*-tikv-......|.*(tidb|worker|scheduling|pd|tso|cluster-agent|tibr|tibrgc|restore|tiflash|tiproxy|o11y|cdc).*"}) by (namespace,pod)
              * 100
            )[24h:]
          ) < 45
        for: 1h
        labels:
          severity: warning
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          tier: next-gen

      - alert: NextGenMemUsageIsTooLow
        annotations:
          message: Memory utilization of pod {{ $labels.pod }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} is less than 45% for recent 24 hours.
          summary: Pod memory usage is idle, consider reducing resource allocation
        expr: |
          max_over_time(
            (
              sum(container_memory_working_set_bytes{namespace=~"tidb[^-].*",container!="POD", container!="", image!="", pod!~".*-tikv-......|.*(tidb|worker|scheduling|pd|tso|cluster-agent|tibr|tibrgc|restore|tiflash|tiproxy|o11y|cdc).*"})by(pod)/
              sum(kube_pod_container_resource_limits{resource="memory",namespace=~"tidb[^-].*", pod!~".*-tikv-......|.*(tidb|worker|scheduling|pd|tso|cluster-agent|tibr|tibrgc|restore|tiflash|tiproxy|o11y|cdc).*"}) by (pod)
              * 100
            )[24h:]
          ) < 45
        for: 1h
        labels:
          severity: warning
          component: ng-auto-scaler
          assign: lu.xing@pingcap.com
          oncall_team: premium
          tier: next-gen