groups:
- name: scheduling
  rules:
  - alert: SchedulingServerDowntimeIsTooLong
    expr: up{component="scheduling", job=~".*schedul.*"} == 0
    for: 10m
    labels:
      severity: critical
      response_class: page
      tier: next-gen
      component: pd
      oncall_team: pd
    annotations:
      description: 'Scheduling server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has been down for more than 10 minutes. The scheduling service is critical for region management and load balancing. Please immediately check and restart the scheduling server to restore cluster scheduling functionality. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: Scheduling server downtime is more than 10 min
  - alert: SchedulingServerFrequentlyRestart
    expr: |-
      sum by (cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*schedul.*"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*schedul.*"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      oncall_team: pd
    annotations:
      description: 'Scheduling server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has restarted {{ $value }} times in the past hour. Frequent scheduling server restarts can affect region scheduling and cluster balance. Please investigate the root cause and ensure scheduling server stability. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      summary: Scheduling server restarted {{ $value }} times in past an hour
  - alert: SchedulingClusterFrequentlyRestart
    expr: |-
      (
        sum by (cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*schedul.*"}))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*schedul.*"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*schedul.*"}))[1h:15s])
            ) by (cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      oncall_team: pd
    annotations:
      description: 'Multiple scheduling servers in cluster {{ $labels.cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting scheduling service stability. Please investigate the cluster infrastructure and scheduling configuration to prevent further restarts. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      summary: Scheduling servers in a cluster restarted {{ $value }} times in past an hour
  - alert: SchedulingServerDiscoverLowSpaceStore
    expr: (sum ( pd_cluster_status{type="store_low_space_count"} ) by (cluster_id, instance, tenant, provider_type) > 0) and (sum(service_member_role{service="Scheduling Service"}) by (cluster_id, instance, tenant, provider_type) > 0)
    for: 5m
    labels:
      severity: warning
      tier: next-gen
      component: pd
      oncall_team: tikv_unified
    annotations:
      description: 'Scheduling server in cluster {{ $labels.cluster_id }} has detected {{ $value }} TiKV stores with low disk space. This may affect data writing and cluster performance. Please monitor disk usage and consider expanding storage capacity or adding more TiKV nodes to prevent storage issues. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: "Scheduling server found some store space is too low"