groups:
- name: pd/tso-server
  rules:
  - alert: TsoServerHighCPU
    expr: (sum(rate(container_cpu_usage_seconds_total{namespace="tidb-serverless", pod=~"pd-tso-server-.*", container!="POD", container!=""}[2m])) by (pod) / sum(kube_pod_container_resource_limits{namespace="tidb-serverless", pod=~"pd-tso-server-.*", resource="cpu"}) by (pod))*100 > 70
    for: 2m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'CPU utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 70% for 2m'
      summary: "Tso Server CPU usage exceeds the threshold"
  - alert: TsoServerCPUThrottled
    expr: (sum(rate(container_cpu_cfs_throttled_periods_total{namespace="tidb-serverless", pod=~"pd-tso-server-.*", container!="POD", container!=""}[2m])) by (pod) / sum(rate(container_cpu_cfs_periods_total{namespace="tidb-serverless", pod=~"pd-tso-server-.*", container!="POD", container!=""}[2m])) by (pod))*100 > 30
    for: 2m
    labels:
      severity: critical
      response_class: page
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'TSO Server {{ $labels.pod }}, CPU throttle ratio is {{ $value | humanizePercentage }}, exceeds 30% threshold.'
      runbook_url:
      value: '{{ $value }}'
      summary: "TSO Server CPU throttling exceeds the threshold"
  - alert: TsoServerHighMemory
    expr: (sum(container_memory_working_set_bytes{namespace="tidb-serverless", pod=~"pd-tso-server-.*", container!="POD", container!="", image!=""}) by (namespace, pod) / sum(kube_pod_container_resource_limits{namespace="tidb-serverless", pod=~"pd-tso-server-.*", resource="memory"}) by (namespace, pod))*100 > 70
    for: 2m
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      message: 'Memory utilization of TSO Server {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 70% for 2m'
      runbook_url:
      value: '{{ $value }}'
      summary: "TSO Server memory usage exceeds the threshold"
  - alert: TsoServerLeaderChange
    expr: sum(pd_tso_role{group="0"} != (pd_tso_role{group="0"} offset 5m)) > 0
    labels:
      severity: warning
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
      expr: sum(pd_tso_role{group="0"} != (pd_tso_role{group="0"} offset 5m)) > 0
    annotations:
      summary: "TSO Server leader has changed"
      message: 'TSO has changed its leader status'
  - alert: TsoServerLeaderChangeWithSlaDecline
    expr: sum((pd_tso_role{group="0"} != (pd_tso_role{group="0"} offset 5m)) and on() (count((1 - (sum(increase(gateway_connection_sla_total{cluster_id!="",status=~"inner_error|tidb_error|cancel"}[1m])) by (tidb_pool, cluster_id)/sum(increase(gateway_connection_sla_total{cluster_id!="",status!="missmatch"}[1m])) by (tidb_pool, cluster_id))) < 0.90) > 0)) > 0
    labels:
      severity: major
      response_class: ticket
      tier: serverless
      provider_type: aws-free-tier
      component: pd
      oncall_team: serverless
    annotations:
      summary: "TSO Server leader has changed"
      message: 'TSO has changed its leader status'