groups:
- name: serverless.resource-waste.rules
  rules:
  - alert: TiDBNodeGroupCPUResourceWaste
    annotations:
      message: TiDB nodes have {{ printf "%.2f" $value }} excess CPU cores beyond twice the requested CPU for over 1 hour. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=us-east-1-f06%2Faws-us-east-1%2Fdevtier-infra&var-node_group=All&var-node=All&var-ns=tidb-serverless&var-pod=All&var-node_type=All&from=now-3h&to=now)
    expr: |
      sum(
        kube_node_status_capacity{resource="cpu"} * on(node) group_left(label_serverless_tidbcloud_com_node, label_eks_amazonaws_com_capacity_type)
        count(:node_cpu_idle:kube_node_labels{label_serverless_tidbcloud_com_node="tidb"}) by (label_serverless_tidbcloud_com_node, label_eks_amazonaws_com_capacity_type, node)
      ) - 2 * sum(kube_pod_container_resource_requests{resource="cpu", namespace=~".*tidb-pool"}) > 0
    for: 1h
    labels:
      severity: warning
      component: k8s-resource
      oncall_team: serverless
  - alert: TiKVNodeGroupMemoryResourceWaste
    annotations:
      message: TiKV nodes have {{ printf "%.2f" $value }} GiB of excess memory capacity for over 1 hour. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=us-east-1-f06%2Faws-us-east-1%2Fdevtier-infra&var-node_group=All&var-node=All&var-ns=tidb-serverless&var-pod=All&var-node_type=All&from=now-3h&to=now)
    expr: |
      sum(
        kube_node_status_capacity{resource="memory"} * on(node) group_left(label_serverless_tidbcloud_com_node, label_eks_amazonaws_com_capacity_type)
        count(:node_cpu_idle:kube_node_labels{label_serverless_tidbcloud_com_node="tikv"}) by (label_serverless_tidbcloud_com_node, label_eks_amazonaws_com_capacity_type, node)
      ) / 1024 / 1024 / 1024 - (
        sum(kube_pod_container_resource_limits{resource="memory", namespace="tidb-serverless", container="tikv"}) / 1024 / 1024 / 1024 +
        count(max_over_time((1 - :node_cpu_idle:kube_node_labels{label_serverless_tidbcloud_com_node="tikv"})[1m:]) > 0) * 8
      ) > 0
    for: 1h
    labels:
      severity: warning
      component: k8s-resource
      oncall_team: serverless
  - alert: CriticalServicesNodeGroupCPUResourceWaste
    annotations:
      message: Critical-services nodes have {{ printf "%.2f" $value }} excess CPU cores beyond twice the requested CPU for over 1 hour. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=us-east-1-f06%2Faws-us-east-1%2Fdevtier-infra&var-node_group=All&var-node=All&var-ns=tidb-serverless&var-pod=All&var-node_type=All&from=now-3h&to=now)
    expr: |
      sum(
        kube_node_status_capacity{resource="cpu"} * on(node) group_left(label_serverless_tidbcloud_com_node, label_eks_amazonaws_com_capacity_type)
        count(:node_cpu_idle:kube_node_labels{label_serverless_tidbcloud_com_node="critical-services"}) by (label_serverless_tidbcloud_com_node, label_eks_amazonaws_com_capacity_type, node)
      ) - 2 * sum(kube_pod_container_resource_requests{resource="cpu", namespace=~"tidb-serverless|tidb-admin", container=~"pd|pd-tso-server|etcd|tidb-gateway"}) > 10
    for: 24h
    labels:
      severity: warning
      component: k8s-resource
      oncall_team: serverless
  - alert: NodeLowResourceAllocation
    annotations:
      message: "Node {{ $labels.node }} (nodetype={{ $labels.label_serverless_tidbcloud_com_node }}, nodegroup={{ $labels.label_serverless_tidbcloud_com_nodegroup }}, instance_type={{ $labels.label_node_kubernetes_io_instance_type }}) has CPU and memory allocation below 30% for over 24 hours (CPU allocation: {{ printf \"%.2f\" $value }}). [Diagnosis Board](https://clinic.pingcap.com/grafana/d/KmTtYdBNk/serverless-resource-usage?orgId=1&var-control_plane_info=us-east-1-f06%2Faws-us-east-1%2Fdevtier-infra&var-node_group=All&var-node=All&var-ns=tidb-serverless&var-pod=All&var-node_type=All&from=now-24h&to=now&viewPanel=64&editPanel=64)"
    expr: |
      (
        sum(
          kube_pod_container_resource_requests{resource="cpu"}
          * on(pod, namespace) group_left()
          sum by (pod, namespace) (kube_pod_status_phase{phase=~"Running|Pending"} == 1)
        ) by (node)
        /
        sum(kube_node_status_allocatable{resource="cpu"}) by (node)
      )
      * on(node) group_left(
        label_serverless_tidbcloud_com_node,
        label_serverless_tidbcloud_com_nodegroup,
        label_node_kubernetes_io_instance_type
      )
      max by (
        node,
        label_serverless_tidbcloud_com_node,
        label_serverless_tidbcloud_com_nodegroup,
        label_node_kubernetes_io_instance_type
      ) (kube_node_labels) < 0.3
      and
      (
        sum(
          kube_pod_container_resource_requests{resource="memory"}
          * on(pod, namespace) group_left()
          sum by (pod, namespace) (kube_pod_status_phase{phase=~"Running|Pending"} == 1)
        ) by (node)
        /
        sum(kube_node_status_allocatable{resource="memory"}) by (node)
      )
      * on(node) group_left(
        label_serverless_tidbcloud_com_node,
        label_serverless_tidbcloud_com_nodegroup,
        label_node_kubernetes_io_instance_type
      )
      max by (
        node,
        label_serverless_tidbcloud_com_node,
        label_serverless_tidbcloud_com_nodegroup,
        label_node_kubernetes_io_instance_type
      ) (kube_node_labels) < 0.3
    for: 24h
    labels:
      severity: warning
      component: k8s-resource
      oncall_team: serverless