groups:
  - name: mgmt.dedicated-global-service
    rules:
      - alert: ClusterMeterLagVeryHigh
        annotations:
          message: Cluster metering lag {{ $value | humanizeDuration }} cluster_id={{ $labels.cluster_id }}
          diagnosis: click https://pingcap-cn.feishu.cn/wiki/Cittw3iTTiqJ8vksvHwcpOafnXf to get more info
        expr: |
          global_dedicated_service_metering_meter_lag_seconds{meter_type="cluster"} > 10800
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
          service: dedicated-global-service

      - alert: ReportLagHigh
        annotations:
          message: Metering report lag {{ $value | humanizeDuration }}
          diagnosis: click https://pingcap-cn.feishu.cn/wiki/Cittw3iTTiqJ8vksvHwcpOafnXf to get more info
        expr: |
          global_dedicated_service_metering_report_lag_seconds{} > 3600
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
          service: dedicated-global-service

      - alert: ReportLagVeryHigh
        annotations:
          message: Metering report lag {{ $value | humanizeDuration }}
          diagnosis: click https://pingcap-cn.feishu.cn/wiki/Cittw3iTTiqJ8vksvHwcpOafnXf to get more info
        expr: |
          global_dedicated_service_metering_report_lag_seconds{} > 10800
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
          service: dedicated-global-service

      - alert: ClusterMeterFailRateHigh
        annotations:
          message: Cluster metering failure rate {{ printf "%.0f" $value }}% cluster_id={{ $labels.cluster_id }}
          diagnosis: click https://pingcap-cn.feishu.cn/wiki/Cittw3iTTiqJ8vksvHwcpOafnXf to get more info
        expr: |
          (100-(sum(rate(global_dedicated_service_metering_meter_total{cluster_id!="0", meter_type="cluster", success="true"}[10m])) BY (cluster_id) 
            / sum(rate(global_dedicated_service_metering_meter_total{cluster_id!="0", meter_type="cluster"}[10m])) BY (cluster_id) 
          ) * 100) > 40
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
          service: dedicated-global-service

      - alert: ClusterMeterP95Slow
        annotations:
          message: Cluster metering p95 {{ $value | humanizeDuration }}
          diagnosis: click https://pingcap-cn.feishu.cn/wiki/Cittw3iTTiqJ8vksvHwcpOafnXf to get more info
        expr: |
          histogram_quantile(0.95,
            sum(rate(global_dedicated_service_metering_meter_duration_seconds_bucket{
              cluster_id="0", meter_type="cluster"
          }[5m])) by (le)) > 60
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
          service: dedicated-global-service