groups:
- name: pd/keyspace-group
  rules:
  - alert: EtcdGroupHighCPU
    expr: ((sum(rate(container_cpu_usage_seconds_total{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*", container!="POD", container!=""}[2m])) by (pod) / sum(kube_pod_container_resource_limits{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*", resource="cpu"}) by (pod)) or on(pod) vector(0)) > 0.7
    for: 2m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, instance: {{ $labels.instance }}'
      runbook_url:
      value: '{{ $value }}'
      summary: "CPU usage exceeds the threshold"
  - alert: EtcdGroupCPUThrottled
    expr: ((sum(rate(container_cpu_cfs_throttled_periods_total{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*", container!="POD", container!=""}[2m])) by (pod) / sum(rate(container_cpu_cfs_periods_total{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*", container!="POD", container!=""}[2m])) by (pod)) or on(pod) vector(0)) > 0.3
    for: 2m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, pod: {{ $labels.pod }}, CPU throttle ratio is {{ $value | humanizePercentage }}, exceeds 30% threshold.'
      runbook_url:
      value: '{{ $value }}'
      summary: "Keyspace etcd group CPU throttling exceeds the threshold"
  - alert: EtcdGroupHighMemory
    expr: ((sum(container_memory_working_set_bytes{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*", container!="POD", container!="", image!=""}) by (namespace, pod) / sum(kube_pod_container_resource_limits{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*", resource="memory"}) by (namespace, pod)) or on(namespace, pod) vector(0)) > 0.7
    for: 2m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'region: {{ $labels.region }}, pod: {{ $labels.pod }}, keyspace etcd group memory usage is {{ $value | humanizePercentage }}, exceeds 70% threshold.'
      runbook_url:
      value: '{{ $value }}'
      summary: "Keyspace etcd group memory usage exceeds the threshold"
  - alert: EtcdGroupHighDisk
    expr: |
      (
        kubelet_volume_stats_used_bytes{namespace="tidb-serverless", persistentvolumeclaim=~"datadir-keyspace-etcd-group-.*"} /
        kubelet_volume_stats_capacity_bytes{namespace="tidb-serverless", persistentvolumeclaim=~"datadir-keyspace-etcd-group-.*"}
      )
      * on (namespace, persistentvolumeclaim) group_left(pod)
      kube_pod_spec_volumes_persistentvolumeclaims_info{namespace="tidb-serverless", pod=~"keyspace-etcd-group-.*"}
      > 0.7
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'Keyspace etcd group pod {{ $labels.pod }} pvc {{ $labels.persistentvolumeclaim }} disk usage exceeds 70% ({{ $value }}).'
      runbook_url:
      value: '{{ $value }}'
      summary: "Keyspace etcd group disk usage exceeds the threshold"
  - alert: EtcdGroupMVCCDBTotalSizeTooLarge
    expr: etcd_mvcc_db_total_size_in_bytes{pod=~".*etcd-group.*"} > 6 * 1024 * 1024 * 1024
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'pod: {{ $labels.pod }}, instance: {{ $labels.instance }}, etcd MVCC DB total size is {{ $value }} bytes, exceeds 6 GiB.'
      runbook_url:
      value: '{{ $value }}'
      summary: "Keyspace etcd group MVCC DB total size exceeds 6 GiB"
  - alert: EtcdGroupMVCCDBTotalSizeInUseTooLarge
    expr: etcd_mvcc_db_total_size_in_use_in_bytes{pod=~".*etcd-group.*"} > 4 * 1024 * 1024 * 1024
    for: 5m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'pod: {{ $labels.pod }}, instance: {{ $labels.instance }}, etcd MVCC DB total size in use is {{ $value }} bytes, exceeds 4 GiB.'
      runbook_url:
      value: '{{ $value }}'
      summary: "Keyspace etcd group MVCC DB total size in use exceeds 4 GiB"