groups:
- name: SLOsForAccount
  rules:
  - alert: SLOAccountHttpApiAvailabilityBreakRisk
    annotations:
      message: |
        SLO 'Account HTTP API Availability' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      sum(rate(kong_custom_http_status{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*", path!~"/api_proxy/billing.*", code!~"5.*"}[10m]))
      /
      sum(rate(kong_custom_http_status{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*", path!~"/api_proxy/billing.*"}[10m]))
      < 0.9995
    labels:
      severity: major
      response_class: ticket
      component: account-server
      oncall_team: account
  - alert: SLOAccountHttpApiLatencyP99BreakRisk
    annotations:
      message: |
        SLO 'Account HTTP API Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*", le="5000"}[1h]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*", le="5000"}[5m]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: account-server
      oncall_team: account
  - alert: SLOAccountHttpApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'Account HTTP API Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*", le="700"}[1h]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(kong_custom_request_latency_ms_bucket{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*", le="700"}[5m]))
        /
        sum(rate(kong_custom_request_latency_ms_count{kube_service=~"(nightly-ms|staging-ms|prod-ms).account-central-http.*"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: account-server
      oncall_team: account
  - alert: SLOAccountGrpcApiAvailabilityBreakRisk
    annotations:
      message: SLO 'Account GRPC API Availability' Break Risk
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: account-server
      oncall_team: account
  - alert: SLOAccountGrpcApiLatencyP99BreakRisk
    annotations:
      message: |
        SLO 'Account GRPC API Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary", le="5"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary", le="5"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: account-server
      oncall_team: account
  - alert: SLOAccountGrpcApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'Account GRPC API Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary", le="0.5"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary", le="0.5"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"account-provider.*", grpc_type="unary"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: account-server
      oncall_team: account