groups:
  - name: core-server
    rules:
      - alert: CoreServerRPCSystemErrorRateHigh
        expr: |
          (
            sum by (namespace, method) (
              rate(
                devops_core_server_rpc_request_duration_seconds_count{
                  code=~"Unknown|DeadlineExceeded|ResourceExhausted|Unimplemented|Internal|Unavailable|DataLoss"
                }[5m]
              )
            )
            /
            clamp_min(
              sum by (namespace, method) (
                rate(devops_core_server_rpc_request_duration_seconds_count[5m])
              ),
              0.000001
            )
          ) > 0.05
        for: 5m
        labels:
          severity: major
          component: core-server
          oncall_team: resilience
          response_class: ticket
        annotations:
          summary: "core-server RPC system error rate is high"
          description: "namespace: {{ $labels.namespace }}, method: {{ $labels.method }}, system error rate: {{ $value | humanizePercentage }}"

      - alert: CoreServerUserSyncFailed
        expr: |
          sum by (namespace, service_name, job_name) (
            increase(
              devops_core_server_cron_job_sync_count_total{
                service_name="UserService",
                job_name=~"SyncUsers|SyncUserGroups",
                result="failed"
              }[5m]
            )
          ) > 0
        labels:
          severity: major
          component: core-server
          oncall_team: resilience
          response_class: ticket
        annotations:
          summary: "core-server user synchronization failed"
          description: "namespace: {{ $labels.namespace }}, service: {{ $labels.service_name }}, job: {{ $labels.job_name }}, failures in the last 5 minutes: {{ $value | humanize }}"