groups:
  - name: tidb-worker/scaler
    rules:
      # CRITICAL
      - alert: TiDBWorkerScalerNoAvailableReplicas
        expr: |
          kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"} == 0
          or
          absent(kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"})
        for: 6h
        annotations:
          message: "no available tidb worker scaler for more than 6 hour"
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: tidb
          oncall_team: serverless
          expr: kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"} == 0 or absent(kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"})

      # WARNING
      - alert: TiDBWorkerScalerMissing
        expr: |
          kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"} == 0
          or
          absent(kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"})
        for: 5m
        annotations:
          message: "no available tidb worker scaler for more than 5 min"
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"} == 0 or absent(kube_deployment_status_replicas_available{deployment="scaler",namespace="tidb-worker"})

      - alert: TiDBWorkerTooManyPendingTasks
        expr: sum by (pod, type) (tidb_worker_scaler_current_task_counter{namespace="tidb-worker", state="pending"}) > 1000
        for: 15m
        annotations:
          description: 'too many pending {{ $labels.type }} tasks in tidb-worker, currently {{ $value }}'
          summary: 'More than 1000 pending tasks in tidb-worker'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: sum by (pod, type) (tidb_worker_scaler_current_task_counter{namespace="tidb-worker", state="pending"}) > 1000

      - alert: TiDBWorkerTooManyPods
        expr: count(kube_pod_info{namespace="tidb-worker"}) > 100
        for: 5m
        annotations:
          description: 'too many pods running in tidb-worker namespace, currently {{ $value }}'
          summary: 'More than 100 pods running in the tidb-worker namespace'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: count(kube_pod_info{namespace="tidb-worker"}) > 100

      - alert: TiDBWorkerTooManyGCWorkers
        expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-gc.*"}) > 50
        for: 5m
        annotations:
          description: 'too many gc/gcv2 workers in tidb-worker namespace, currently {{ $value }}'
          summary: 'More than 50 GC workers running in the tidb-worker namespace'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-gc.*"}) > 50

      - alert: TiDBWorkerTooManyDDLWorkers
        expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-ddl-.*"}) > 40
        for: 5m
        annotations:
          description: 'too many ddl workers in tidb-worker namespace, currently {{ $value }}'
          summary: 'More than 40 DDL workers running in the tidb-worker namespace'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-ddl-.*"}) > 40

      - alert: TiDBWorkerTooManyBatchWorkers
        expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-batch-.*"}) > 20
        for: 5m
        annotations:
          description: 'too many batch workers in tidb-worker namespace, currently {{ $value }}'
          summary: 'More than 20 batch workers running in the tidb-worker namespace'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-batch-.*"}) > 20

      - alert: TiDBWorkerTooManyTTLWorkers
        expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-ttl-.*"}) > 100
        for: 5m
        annotations:
          description: 'too many ttl workers in tidb-worker namespace, currently {{ $value }}'
          summary: 'More than 100 TTL workers running in the tidb-worker namespace'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-ttl-.*"}) > 50

      - alert: TiDBWorkerTooManyAutoAnalyzeWorkers
        expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-auto-analyze-.*"}) > 20
        for: 5m
        annotations:
          description: 'too many auto analyze workers in tidb-worker namespace, currently {{ $value }}'
          summary: 'More than 20 auto analyze workers running in the tidb-worker namespace'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: count(kube_pod_info{namespace="tidb-worker",pod=~"tidb-worker-auto-analyze-.*"}) > 20

      - alert: TiDBWorkerGCWorkerRunningTooLong
        expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-gc.*"} > 3600
        for: 5m
        annotations:
          description: 'gc/gcv2 worker {{ $labels.pod }} has been running for more than 1 hour'
          summary: 'GC worker has been running for more than 1 hour'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-gc.*"} > 3600

      - alert: TiDBWorkerDDLWorkerRunningTooLong
        expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-ddl-.*"} > 86400
        for: 5m
        annotations:
          description: 'ddl worker {{ $labels.pod }} has been running for more than 1 day'
          summary: 'DDL worker has been running for more than 1 day'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-ddl-.*"} > 86400

      - alert: TiDBWorkerBatchWorkerRunningTooLong
        expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-batch-.*"} > 86400
        for: 5m
        annotations:
          description: 'batch worker {{ $labels.pod }} has been running for more than 1 day'
          summary: 'Batch worker has been running for more than 1 day'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-batch-.*"} > 86400

      - alert: TiDBWorkerSharedWorkerRunningTooLong
        expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-shared-.*"} > 86400
        for: 5m
        annotations:
          description: 'shared worker {{ $labels.pod }} has been running for more than 1 day'
          summary: 'Shared worker has been running for more than 1 day'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-shared-.*"} > 86400

      - alert: TiDBWorkerTTLWorkerRunningTooLong
        expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-ttl-.*"} > 3600
        for: 5m
        annotations:
          description: 'ttl worker {{ $labels.pod }} has been running for more than 1 hour'
          summary: 'TTL worker has been running for more than 1 hour'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-ttl-.*"} > 3600

      - alert: TiDBWorkerAutoAnalyzeWorkerRunningTooLong
        expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-auto-analyze-.*"} > 3600
        for: 5m
        annotations:
          description: 'auto analyze worker {{ $labels.pod }} has been running for more than 1 hour'
          summary: 'Auto analyze worker has been running for more than 1 hour'
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
          expr: time() - kube_pod_start_time{namespace="tidb-worker",pod=~"tidb-worker-auto-analyze-.*"} > 3600
      - alert: TiDBWorkerScalerHighCPUUsage
        annotations:
          message: |
            cluster {{ $labels.cluster }}, Scaler CPU usage is high (>{{ $value | humanizePercentage }}) for pod {{ $labels.pod }}.
        expr: |
          rate(container_cpu_usage_seconds_total{namespace="tidb-worker",pod=~"scaler-.*",image=""}[1m]) / on(pod) kube_pod_container_resource_limits{resource="cpu",namespace="tidb-worker",pod=~"scaler-.*"} > 0.8
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless
      - alert: TiDBWorkerScalerHighMemoryUsage
        annotations:
          message: |
            cluster {{ $labels.cluster }}, Scaler memory usage is high (>{{ $value | humanizePercentage }}) for pod {{ $labels.pod }}.
        expr: |
          container_memory_working_set_bytes{pod=~"scaler-.*",container!="POD",container!="",image!="",namespace="tidb-worker"}/ on(pod) kube_pod_container_resource_limits{resource="memory",namespace="tidb-worker",pod=~"scaler-.*"} > 0.8
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: tidb-worker
          oncall_team: serverless