groups:
  - name: probe.rules
    rules:
      - alert: ServerlessSelfProbeLatencyIsTooLong
        annotations:
          message: region={{ $labels.region }}, user={{ $labels.user }}, sql_type={{ $labels.stmt }} probe latency is {{ $value }}ms, longer then 1s for 5 minutes. [Probe Latency Board](https://clinic.pingcap.com/grafana/d/A_nBs2ASz/serverless-observe-latency?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-stmt=All&var-user=All&from=now-1h&to=now)
        expr: |
          (histogram_quantile(0.95, sum(rate(gateway_probe_latency_v1_seconds_bucket{status="OK"}[1m])) by (region, user, stmt, le))) > 1
        for: 5m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessSelfProbeSuccessRateIsTooLow
        annotations:
          message: region={{ $labels.region }}, sql_type={{ $labels.stmt }} probe success rate is {{ $value }}%, lower than 10% for 5 minutes. [Probe Latency Board](https://clinic.pingcap.com/grafana/d/A_nBs2ASz/serverless-observe-latency?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-stmt=All&var-user=All&from=now-1h&to=now)
        expr: |
          (sum(increase(gateway_probe_latency_v1_seconds_count{status="OK"}[1m])) by (stmt,region) / sum(increase(gateway_probe_latency_v1_seconds_count{}[1m])) by (stmt,region))*100 < 10
        for: 5m
        labels:
          tier: serverless
          severity: warning
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless
      - alert: ServerlessSelfProbeSuccessRateIsTooLow
        annotations:
          message: region={{ $labels.region }}, sql_type={{ $labels.stmt }} probe success rate is {{ $value }}%, lower than 10% for 1 hour. [Probe Latency Board](https://clinic.pingcap.com/grafana/d/A_nBs2ASz/serverless-observe-latency?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-stmt=All&var-user=All&from=now-1h&to=now)
        expr: |
          (sum(increase(gateway_probe_latency_v1_seconds_count{status="OK"}[1m])) by (stmt,region) / sum(increase(gateway_probe_latency_v1_seconds_count{}[1m])) by (stmt,region))*100 < 10
        for: 1h
        labels:
          tier: serverless
          severity: major
          response_class: ticket
          provider_type: aws-free-tier
          component: gateway
          oncall_team: serverless