groups:
- name: PionexCustomizedAlert
  rules:
  - alert: PionexPDClientDurationTooLong
    expr: histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{tenant="1372813089209213994", type!~"tso|tso_async_wait"}[2m])) by (cluster_id, le, type)) > 0.1
    for: 10m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
    annotations:
      message: |
        Customized alert for Pionex PD Client Duration long.