groups:
- name: SLOsForCoreSvc
  rules:
  - alert: SLOCoreSvcGrpcApiAvailabilityBreak
    annotations:
      message: SLO 'Core Svc GRPC API Availability' Break
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd"}[10m]))
      < 0.999
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcGrpcApiAvailabilityBreakRisk
    annotations:
      message: SLO 'Core Svc GRPC API Availability' Break Risk
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcGrpcApiLatencyP99Break
    annotations:
      message: SLO 'Core Svc GRPC API Latency P99' Break
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster|ExecuteCtlCommand"}[10m])) by (le)) > 2
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcGrpcApiLatencyP99BreakRisk
    annotations:
      message: SLO 'Core Svc GRPC API Latency P99' Break Risk
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster|ExecuteCtlCommand"}[10m])) by (le)) > 1.8
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcGrpcApiLatencyP90Break
    annotations:
      message: SLO 'Core Svc GRPC API Latency P90' Break
    expr: |
      histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster|ExecuteCtlCommand"}[10m])) by (le)) > 0.9
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcGrpcApiLatencyP90BreakRisk
    annotations:
      message: SLO 'Core Svc GRPC API Latency P90' Break Risk
    expr: |
      histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-svc-server.*", grpc_type="unary",grpc_method!~"UpsertProfileByYaml|CreateDBUser|DeleteDBUser|IsDBUserExisting|ChangeClusterPasswd|CalcClusterFee|CreateCluster|ExecuteCtlCommand"}[10m])) by (le)) > 0.81
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcCreateOrDeleteClusterGrpcApiAvailabilityBreak
    annotations:
      message: SLO 'Core Svc CreateOrDeleteCluster GRPC API Availability' Break
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m]))
      < 0.999
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcCreateOrDeleteClusterGrpcApiAvailabilityBreakRisk
    annotations:
      message: SLO 'Core Svc CreateOrDeleteCluster GRPC API Availability' Break Risk
    expr: |
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary", grpc_code!="UNKNOWN|DEADLINE_EXCEEDED|INTERNAL|UNAVAILABLE|DATA_LOSS",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m]))
      /
      sum(rate(grpc_server_handled_total{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m]))
      < 0.9995
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcCreateOrDeleteClusterGrpcApiLatencyP99Break
    annotations:
      message: SLO 'Core Svc CreateOrDeleteCluster GRPC API Latency P99' Break
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m])) by (le)) > 4.5
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcCreateOrDeleteClusterGrpcApiLatencyP99BreakRisk
    annotations:
      message: SLO 'Core Svc CreateOrDeleteCluster GRPC API Latency P99' Break Risk
    expr: |
      histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m])) by (le)) > 4.05
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcCreateOrDeleteClusterGrpcApiLatencyP90Break
    annotations:
      message: SLO 'Core Svc CreateOrDeleteCluster GRPC API Latency P90' Break
    expr: |
      histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m])) by (le)) > 3.5
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service
  - alert: SLOCoreSvcCreateOrDeleteClusterGrpcApiLatencyP90BreakRisk
    annotations:
      message: SLO 'Core Svc CreateOrDeleteCluster GRPC API Latency P90' Break Risk
    expr: |
      histogram_quantile(0.90, sum(rate(grpc_server_handling_seconds_bucket{namespace=~"nightly-ms|staging-ms|prod-ms", pod=~"dbaas-core-portal-server.*", grpc_type="unary",grpc_method=~"CreateCluster|CreateClusterV2|DeleteCluster"}[10m])) by (le)) > 3.15
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: core-svc
      oncall_team: dedicated_service