groups:
  - name: load-data-worker.rules
    rules:
      - alert: LoadDataWorkerIsDown
        annotations:
          message: |
            {{ $labels.region }} Serverless load-data-worker is down.
          summary: "Load data worker is down."
          region: '{{ $labels.region }}'
        expr: |
          up{component="load-data-worker"} == 0
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerFrequentlyRestart
        expr: sum(changes(process_start_time_seconds{component="load-data-worker"}[30m])) by (pod, region) >=4
        for: 5m
        labels:
          tier: serverless
          provider_type: aws-free-tier
          severity: critical
          response_class: page
          expr: sum(changes(process_start_time_seconds{component="load-data-worker"}[30m])) by (pod, region) >=4
          component: tikv
          oncall_team: serverless
        annotations:
          message: 'region: {{ $labels.region }}, instance: {{ $labels.pod }}, values: {{ $value }}'
          value: '{{ $value }}'
          summary: Load data worker is down for at least 4 times in half an hour
      - alert: LoadDataWorkerPodPendingTooLong
        annotations:
          message: |
            Serverless load-data-worker pod {{ $labels.namespace }}/{{ $labels.pod }} has been pending for more than 20 minutes.
          summary: "Serverless load-data-worker pod pending too long."
        expr: |
          max by (namespace, pod) (
            kube_pod_status_phase{phase=~"Pending|Unknown", namespace="tidb-serverless", pod=~"ldw.*|load-data-worker.*"}
          ) > 0
        for: 20m
        labels:
          tier: serverless
          severity: major
          response_class: ticket
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerPodRunningTooLong
        annotations:
          message: |
            Serverless load-data-worker pod {{ $labels.namespace }}/{{ $labels.pod }} has been running for more than 3.5 hours.
          summary: "Serverless load-data-worker pod running too long."
        expr: |
          time() - kube_pod_start_time{namespace="tidb-serverless", pod=~"ldw-.*", pod!~"ldw-shared-[0-9]+"} > 12600
        for: 5m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerSplitRegionTooManyRetires
        annotations:
          message: |
            {{ $labels.region }} Serverless load data worker {{ $labels.pod }} split region too many retries
          summary: "{{ $labels.region }} Serverless load data worker {{ $labels.pod }} split region too many retries."
          region: '{{ $labels.region }}'
        expr: |
          tikv_worker_load_data_split_region_failures_counter > 10
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerIngestRangeGroupTooManyRetires
        annotations:
          message: |
            {{ $labels.region }} Serverless load data worker {{ $labels.pod }} ingest range group too many retries
          summary: "{{ $labels.region }} Serverless load data worker {{ $labels.pod }} ingest range group too many retries."
          region: '{{ $labels.region }}'
        expr: |
          tikv_worker_load_data_ingest_range_group_failures_counter > 10
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerGetShardMetaTooManyRetires
        annotations:
          message: |
            {{ $labels.region }} Serverless load data worker {{ $labels.pod }} get shard meta  too many retries
          summary: "{{ $labels.region }} Serverless load data worker {{ $labels.pod }} get shard meta too many retries."
          region: '{{ $labels.region }}'
        expr: |
          tikv_worker_load_data_get_shard_meta_failures_counter > 10
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerDiskUsageHigh
        annotations:
          message: |
            Serverless load data worker pod {{ $labels.pod }} pvc {{ $labels.persistentvolumeclaim }} disk usage exceeds 70% ({{ $value }}).
          summary: "Serverless load data worker {{ $labels.pod }} disk usage high."
        expr: |
          (
            kubelet_volume_stats_used_bytes{ namespace="tidb-serverless", persistentvolumeclaim=~".*ldw.*"} /
            kubelet_volume_stats_capacity_bytes{ namespace="tidb-serverless", persistentvolumeclaim=~".*ldw.*"}
          )
          * on (namespace,persistentvolumeclaim) group_left(pod)
          kube_pod_spec_volumes_persistentvolumeclaims_info{ namespace="tidb-serverless", pod=~".*ldw.*"}
          > 0.7
        for: 1m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless
      - alert: LoadDataWorkerDiskUsageCritical
        annotations:
          message: |
            Serverless load data worker pod {{ $labels.pod }} pvc {{ $labels.persistentvolumeclaim }} disk usage exceeds 90% ({{ $value }}).
          summary: "Serverless load data worker {{ $labels.pod }} disk usage critical."
        expr: |
          (
            kubelet_volume_stats_used_bytes{ namespace="tidb-serverless", persistentvolumeclaim=~".*ldw.*"} /
            kubelet_volume_stats_capacity_bytes{ namespace="tidb-serverless", persistentvolumeclaim=~".*ldw.*"}
          )
          * on (namespace,persistentvolumeclaim) group_left(pod)
          kube_pod_spec_volumes_persistentvolumeclaims_info{ namespace="tidb-serverless", pod=~"ldw.*"}
          > 0.9
        for: 1m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: tikv
          oncall_team: serverless