groups:
- name: SLOsForServerless
  rules:
  - alert: SLOServerlessGrpcApiAvailabilityBreakRisk
    annotations:
      message: SLO 'Serverless GRPC API Availability' Break Risk
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"serverless-svc-grpc-server.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS",grpc_method="SetSpendLimit"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"serverless-svc-grpc-server.*", grpc_type="unary",grpc_method="SetSpendLimit"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: serverless-svc
      oncall_team: serverless
  - alert: SLOServerlessGrpcSetSpendLimitApiLatencyP99BreakRisk
    annotations:
      message: SLO 'Serverless GRPC SetSpendLimit API Latency P99' Break Risk
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"serverless-svc-grpc-server.*", grpc_type="unary",grpc_method="SetSpendLimit"}[10m])) by (le)) > 2.25
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: serverless-svc
      oncall_team: serverless
  - alert: SLOServerlessGrpcSetSpendLimitApiLatencyP90BreakRisk
    annotations:
      message: SLO 'Serverless GRPC SetSpendLimit API Latency P90' Break Risk
    expr: |
      histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"serverless-svc-grpc-server.*", grpc_type="unary",grpc_method="SetSpendLimit"}[10m])) by (le)) > 1.98
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: serverless-svc
      oncall_team: serverless