# use infra-layer metrics to alert such case
groups:
- name: regional-tcidm-server-api
  rules:

  - alert: SomeHTTP5xxErrors
    expr: |
      sum(increase(tcidm_regional_http_request_duration_ms_count{status=~"5.."}[5m])) by (method,path) >= 10
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The HTTP Request {{ $labels.method }} {{ $labels.path }} failed more than 10 times in 5 minutes

  - alert: TooManyHTTP5xxErrors
    expr: |
      sum(increase(tcidm_regional_http_request_duration_ms_count{status=~"5.."}[5m])) by (method,path) >= 20
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The HTTP Request {{ $labels.method }} {{ $labels.path }} failed more than 20 times in 5 minutes

  - alert: TooLowSLA999
    expr: |
      ((sum(increase(tcidm_regional_http_request_duration_ms_count{status=~"2.."}[5m]))
        / sum(increase(tcidm_regional_http_request_duration_ms_count{}[5m]))) < 0.999)
        and
      (sum(increase(tcidm_regional_http_request_duration_ms_count{status=~"2.."}[5m])) > 100)
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The HTTP Server SLA below 999 in 5 minutes

  - alert: TooLowSLA99
    expr: |
      ((sum(increase(tcidm_regional_http_request_duration_ms_count{status=~"2.."}[5m]))
        / sum(increase(tcidm_regional_http_request_duration_ms_count{}[5m]))) < 0.99)
        and
      (sum(increase(tcidm_regional_http_request_duration_ms_count{status=~"2.."}[5m])) > 10)
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The HTTP Server SLA below 99 in 5 minutes

  - alert: TooSlowHTTPRequest
    expr: |
      histogram_quantile(0.99,
        sum(rate(tcidm_regional_http_request_duration_ms{}[5m])) by (method,path,le)
      ) >= 10000
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The HTTP Request {{ $labels.method }} {{ $labels.path }} T99 cost time exceeds 10 seconds in 5 minutes

  - alert: SomeDownstreamErrors
    expr: |
      sum(increase(tcidm_regional_downstream_request_duration_ms_count{ok="false"}[5m])) by (service,action) >= 10
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The Downstream Request {{ $labels.downstream }} {{ $labels.action }} on {{ $labels.service }} failed more than 10 times in 5 minutes

  - alert: TooManyDownstreamErrors
    expr: |
      sum(increase(tcidm_regional_downstream_request_duration_ms_count{ok="false"}[5m])) by (service,action) >= 20
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The Downstream Request {{ $labels.downstream }} {{ $labels.action }} on {{ $labels.service }} failed more than 20 times in 5 minutes

  - alert: TooSlowDownstreamRequest
    expr: |
      histogram_quantile(0.99,
        sum(rate(tcidm_regional_downstream_request_duration_ms{}[5m])) by (service,action,le)
      ) >= 10000
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: tcidm-regional
      oncall_team: tidb_x_tools
    annotations:
      message: The Downstream Request {{ $labels.downstream }} {{ $labels.action }} on {{ $labels.service }} T99 cost time exceeds 10 seconds in 5 minutes