groups:
  - name: etcd.rules
    rules:
      - alert: ServerlessEtcdIsDown
        annotations:
          message: "etcd of serverless tier is down."
          kubernetes: '{{ $labels.cluster }}'
        expr: |
          up{provider_type=~"aws-free-tier|alicloud-serverless", job="etcd-svc"} == 0
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: etcd
          oncall_team: serverless
      - alert: ServerlessEtcdMembersIsDown
        annotations:
          message: "serverless etcd cluster {{ $labels.pod }}: members are down ({{ $value }})."
          kubernetes: '{{ $labels.cluster }}'
        expr: |-
          max by (cluster,pod) (
            sum by (cluster,pod) (up{job="etcd-svc", namespace="tidb-admin"} == bool 0)
          or
            count by (cluster,pod,endpoint) (
              sum by (cluster,pod,endpoint,To) (rate(etcd_network_peer_sent_failures_total{job="etcd-svc",namespace="tidb-admin"}[3m])) > 0.01
            )
          )
          > 0
        for: 5m
        labels:
          component: etcd
          oncall_team: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          tier: serverless
      - alert: ServerlessEtcdNoLeader
        annotations:
          message: "serverless etcd cluster {{ $labels.job }}: member {{ $labels.instance }} has no leader."
          kubernetes: '{{ $labels.cluster }}'
        expr: etcd_server_has_leader{job="etcd-svc", namespace="tidb-admin"} == 0
        for: 1m
        labels:
          component: etcd
          oncall_team: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          tier: serverless
      - alert: ServerlessEtcdInsufficientMembers
        annotations:
          message: "serverless etcd cluster {{ $labels.pod }}: insufficient members ({{ $value }})."
          kubernetes: '{{ $labels.cluster }}'
        expr: sum(up{job="etcd-svc", namespace="tidb-admin"} == bool 1) by (cluster,pod) < ((count(up{job="etcd-svc", namespace="tidb-admin"}) by (cluster,pod) + 1) / 2)
        for: 3m
        labels:
          component: etcd
          oncall_team: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          tier: serverless
      # skip rule validation
      # ignore_validations: rateBeforeAggregation
      - alert: ServerlessEtcdHighNumberOfLeaderChanges
        annotations:
          message: "serverless etcd cluster {{ $labels.job }}: {{ $value }} leader changes within the last 15 minutes. Frequent elections may be a sign of insufficient resources, high network latency, or disruptions by other components and should be investigated."
          kubernetes: '{{ $labels.cluster }}'
        expr: increase((max by (cluster,job) (etcd_server_leader_changes_seen_total{job="etcd-svc", namespace="tidb-admin"}) or 0*absent(etcd_server_leader_changes_seen_total{job="etcd-svc", namespace="tidb-admin"}))[15m:1m]) >= 3
        for: 5m
        labels:
          component: etcd
          oncall_team: serverless
          severity: warning
          provider_type: aws-free-tier
          tier: serverless
      - alert: ServerlessEtcdHighCommitDurations
        annotations:
          message: "serverless etcd cluster {{ $labels.job }}: 99th percentile commit durations {{ $value }} on etcd instance {{ $labels.instance }}."
          kubernetes: '{{ $labels.cluster }}'
        expr: |-
          histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job="etcd-svc", namespace="tidb-admin"}[5m]))
          > 0.25
        for: 10m
        labels:
          component: etcd
          oncall_team: serverless
          severity: warning
          provider_type: aws-free-tier
          tier: serverless
      - alert: ServerlessEtcdBackupCronJobNotSucceedRecently
        annotations:
          message: "serverless etcd-backup cronjob not succeed more than 56h."
          kubernetes: '{{ $labels.cluster }}'
        expr: |-
          time() - kube_cronjob_status_last_successful_time{namespace="tidb-admin",cronjob="etcd-backup"} > 56 * 60 * 60
        labels:
          component: etcd-backup
          oncall_team: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          tier: serverless