groups:
  - name: mgmt.eventbus
    rules:
      - alert: PublishFailRateHigh
        expr: |
          (
            sum(rate(publish_time_seconds_count{namespace="tidb-management-service", success="false"}[3m])) by (event_type, job)
            /
            sum(rate(publish_time_seconds_count{namespace="tidb-management-service"}[3m])) by (event_type, job)
          )
          * 100          
          > 5
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
        annotations:
          message: Publish failure rate {{ printf "%.2f" $value }}% event_type={{ $labels.event_type }} job={{ $labels.job }}

      - alert: ConsumeFailRateHigh
        expr: |
          (
            sum(rate(events_received_latency_seconds_count{namespace="tidb-management-service", success="false"}[3m])) by (event_type, group, job)
            /
            sum(rate(consume_time_seconds_count{namespace="tidb-management-service"}[3m])) by (event_type, group, job)
          )
          * 100          
          > 5
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
        annotations:
          message: Consume failure rate {{ printf "%.2f" $value }}% event_type={{ $labels.event_type }} group={{ $labels.group }} job={{ $labels.job }}