groups:
- name: tso
  rules:
  - alert: TSOServerDowntimeIsTooLong
    expr: up{component="tso", job=~".*tso.*"} == 0
    for: 10m
    labels:
      severity: critical
      response_class: page
      tier: next-gen
      component: pd
      oncall_team: pd
    annotations:
      description: 'TSO server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has been down for more than 10 minutes. TSO service is critical for providing globally unique timestamps. Please immediately check and restart the TSO server to restore timestamp allocation functionality. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: TSO server downtime is more than 10 min
  - alert: TSOServerFrequentlyRestart
    expr: |-
      sum by (cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{job=~".*tso.*"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{job=~".*tso.*"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      oncall_team: pd
    annotations:
      description: 'TSO server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has restarted {{ $value }} times in the past hour. Frequent TSO server restarts can affect timestamp allocation and cluster performance. Please investigate the root cause and ensure TSO server stability. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      summary: TSO server restarted {{ $value }} times in past an hour
  - alert: TSOClusterFrequentlyRestart
    expr: |-
      (
        sum by (cluster_id, tenant, provider_type) (
            resets((-floor(process_start_time_seconds{job=~".*tso.*"}))[1h:15s])
          and
            max_over_time(process_start_time_seconds{job=~".*tso.*"}[30m] offset 1h) > 0
        ) >= 3
      )
      and
      (
        sum(
          (
            sum(
              resets((-floor(process_start_time_seconds{job=~".*tso.*"}))[1h:15s])
            ) by (cluster_id, instance, tenant, provider_type) >= 2
          )
        ) by (cluster_id, tenant, provider_type) > 0
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: pd
      oncall_team: pd
    annotations:
      description: 'Multiple TSO servers in cluster {{ $labels.cluster_id }} have restarted {{ $value }} times in the past hour. This indicates potential cluster-wide issues affecting TSO service stability. Please investigate the cluster infrastructure and TSO configuration to prevent further restarts. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      summary: TSO servers in a cluster restarted {{ $value }} times in past an hour
  # Disable the following rule for ci check
  # ignore_validations: expressionDoesNotUseRangeShorterThan
  - alert: TSOPrimaryChange
    expr: |
      changes(service_member_role{service="TSO Service Group 0"}[15s]) > 0
      unless on(instance)
      (
        (
          sum(
            count_over_time(
              pd_service_maxprocs{component=~".*tso.*"}[30s]
            )
          ) by (instance)
          or on(instance)
          0 * pd_service_maxprocs{component=~".*tso.*"}
        ) <= 1
      )
    labels:
      severity: major
      response_class: ticket
      tier: next-gen
      component: pd
      oncall_team: pd
    annotations:
      description: 'TSO primary has changed in cluster {{ $labels.cluster_id }}. The new primary is {{ $labels.instance }}. Frequent TSO primary changes may cause performance degradation and affect timestamp allocation. Please investigate the cause of primary changes and ensure TSO cluster stability. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      summary: TSO primary changes, might cause performance degrade.
  - alert: TSOServerGenerateTSOFailed
    expr: increase(pd_tso_events{type="exceeded_max_retry"}[10m]) > 0
    labels:
      severity: major
      response_class: ticket
      tier: next-gen
      component: pd
      oncall_team: pd
    annotations:
      description: 'TSO server {{ $labels.instance }} in cluster {{ $labels.cluster_id }} has failed to generate timestamps after exceeding maximum retry attempts. This is a critical issue that affects all timestamp-dependent operations. Please immediately check TSO server status and resolve the timestamp generation failure. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
      summary: TSO server generate tso failed