groups:
- name: SLOsForDataService
  rules:
  - alert: SLODataServiceGrpcApiAvailabilityBreakRisk
    annotations:
      message: SLO 'DataService GRPC API Availability' Break Risk
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: dataservice
      oncall_team: serverless
  - alert: SLODataServiceGrpcApiLatencyP99BreakRisk
    annotations:
      message: |
        SLO 'DataService GRPC API Latency P99' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary", le="5"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary"}[1h]))) 
        > 16.8*0.01
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary", le="5"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary"}[5m])))
        > 16.8*0.01
      )
    labels:
      severity: major
      response_class: ticket
      component: dataservice
      oncall_team: serverless
  - alert: SLODataServiceGrpcApiLatencyP90BreakRisk
    annotations:
      message: |
        SLO 'DataService GRPC API Latency P90' Break Risk, Runbook: https://clinic.pingcap.com/grafana/d/QIw-n_l4k/slo-global-control-plane?orgId=1
    expr: |
      (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary", le="1"}[1h]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary"}[1h]))) 
        > 2.8*0.1
      ) and (
        1-(sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary", le="1"}[5m]))
        /
        sum(rate(grpc_server_handling_seconds_count{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dataservice-dataapi.*", grpc_type="unary"}[5m])))
        > 2.8*0.1
      )
    labels:
      severity: major
      response_class: ticket
      component: dataservice
      oncall_team: serverless