groups:
- name: k8s
  rules:
  - alert: TiDBOutOfMemory
    # only expose this alarm to specific customer: Hytto(1372813089208341412)
    annotations:
      message: |
        At least one TiDB node in cluster TiDBClusterName in project TiDBProjectName ran out of memory while executing a SQL statement. Consider increasing the memory available to queries using the tidb_mem_quota_query session variable. To monitor node memory utilization, see <a href="https://docs.pingcap.com/tidbcloud/public-preview/monitor-tidb-cluster">Monitoring a TiDB Cluster</a> topic.
      description: At least one TiDB node in the cluster has run out of memory
    expr: label_replace(increase(kube_pod_container_status_restarts_total{tenant=~"1372813089208341412", container="tidb"}[5m]) >= 1 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{tenant=~"1372813089208341412",reason='OOMKilled',container="tidb"} == 1,"cluster_id","$2", "namespace", "(tidb)(.*)")
    labels:
      visibility: external
      severity: major
      component: tidb
  - alert: WetechIOPSTooHighThan8000
    annotations:
      message: Wetech cluster has too high IOPS.
      description: High IOPS for TiFlash.
    expr: |
      label_replace(avg by (label_cluster, label_component, instance) (label_replace(kube_node_labels{label_cluster=~"1379661944646413143|1379661944646413655|1379661944646413999", label_component=~"tiflash"}, "instance", "$1", "node", "(.*)") * on (cluster, instance) group_right (label_cluster, label_component) (sum by (cluster, instance) (rate(node_disk_reads_completed_total{}[2m]) + rate(node_disk_writes_completed_total{}[2m])))) > 8000, "tidb_cluster_id", "$1", "label_cluster", "(.*)")
    for: 5m
    labels:
      visibility: external
      component: resilience
      severity: major
  - alert: WetechIOPSTooHighThan6000
    annotations:
      message: Wetech cluster has too high IOPS.
      description: High IOPS for TiKV.
    expr: |
      label_replace(avg by (label_cluster, label_component, instance) (label_replace(kube_node_labels{label_cluster=~"1379661944646413411|1379661944646414267", label_component=~"tikv"}, "instance", "$1", "node", "(.*)") * on (cluster, instance) group_right (label_cluster, label_component) (sum by (cluster, instance) (rate(node_disk_reads_completed_total{}[2m]) + rate(node_disk_writes_completed_total{}[2m])))) > 6000, "tidb_cluster_id", "$1", "label_cluster", "(.*)")
    for: 5m
    labels:
      visibility: external
      component: resilience
      severity: major
  - alert: EBSElasticVolumeIOPSLimitExceeded
    expr: |
      max by(cluster_id)(
        label_replace(
          (
            max by(namespace)(
              label_replace(
                rate(aws_ebs_csi_exceeded_iops_seconds_total{
                  tenant=~".*1372813089209061633",
                  project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"
                }[2m]),
                "node", "$1", "instance", "(.+)"
              )
              * on(node) group_right()
              kube_pod_info{pod=~"db-tikv-.*"}
            ) > 0.50
          ),
          "cluster_id", "$1", "namespace", "tidb(.*)"
        )
      )
    for: 2m
    labels:
      severity: major
      component: tikv
      visibility: external
      description: EBS IOPS limit has been exceeded for more than 20% of the time on cluster {{ $labels.cluster_id }} for 5 minutes.

      message: >
        EBS IOPS limit has been exceeded for more than 20% of the time on cluster {{ $labels.cluster_id }} for 5 minutes.
        Recommended actions: 1) Contact TiDB Cloud support if the issue persists or latency is impacted. 2) Consider upgrading the storage IOPS configuration for affected TiKV nodes.
    annotations:
      description: EBS IOPS limit has been exceeded for more than 20% of the time on cluster {{ $labels.cluster_id }} for 5 minutes.
      message: >
        EBS IOPS limit has been exceeded for more than 20% of the time on cluster {{ $labels.cluster_id }} for 5 minutes.
        Recommended actions: 1) Contact TiDB Cloud support if the issue persists or latency is impacted. 2) Consider upgrading the storage IOPS configuration for affected TiKV nodes.