groups: 
  - name: clinic.cloud-server
    rules:
      - alert: CloudServer5XXErrorCodeTooMany(30%)
        expr: |
          (
            sum(rate(kong_custom_http_status{code=~"5.*", kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m]))
            /
            sum(rate(kong_custom_http_status{kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m]))
          ) > 0.3
          and (sum(rate(kong_custom_http_status{kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m])) > 5)
        for: 3m
        annotations:
          message: |
            cloud-server 5xx error code too many in 3 minutes, the percent > 30.
            check the grafana dashboard (https://clinic.pingcap.com/grafana/d/mY9p7dQmz/kong) for more details.
        labels:
          severity: critical
          response_class: page
          component: clinic
          oncall_team: clinic
      - alert: CloudServer5XXErrorCodeTooMany(20%)
        expr: |
          (
            sum(rate(kong_custom_http_status{code=~"5.*", kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m]))
            /
            sum(rate(kong_custom_http_status{kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m]))
          ) > 0.2
          and (sum(rate(kong_custom_http_status{kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m])) > 5)
        for: 3m
        annotations:
          message: |
            cloud-server 5xx error code too many in 3 minutes, the percent > 20.
            check the grafana dashboard (https://clinic.pingcap.com/grafana/d/mY9p7dQmz/kong) for more details.
        labels:
          severity: major
          response_class: ticket
          component: clinic
          oncall_team: clinic
      - alert: CloudServerRequestLatencyTooHigh(1s)
        expr: |
          histogram_quantile(0.95, sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"clinic-prod-cloud.cloud-server-service.*"}[1m])) by (le)) >= 1000
        for: 10m
        annotations:
          message: |
            cloud-server request latency p95 is greater than 1s for 10 minutes.
            check the grafana dashboard (https://clinic.pingcap.com/grafana/d/mY9p7dQmz/kong) for more details.
        labels:
          severity: critical
          response_class: page
          component: clinic
          oncall_team: clinic