groups:
  - name: ticket-manager
    rules:
      - alert: TicketManagerHTTPErrorRateHigh
        expr: |
          (
            sum by (namespace, method, url) (
              rate(ticket_manager_gin_request_duration_seconds_count{code=~"5.."}[5m])
            )
            /
            clamp_min(
              sum by (namespace, method, url) (
                rate(ticket_manager_gin_request_duration_seconds_count[5m])
              ),
              0.000001
            )
          ) > 0.05
        for: 5m
        labels:
          severity: major
          component: ticket-manager
          oncall_team: resilience
          response_class: ticket
        annotations:
          summary: "ticket-manager HTTP error rate is high"
          description: "namespace: {{ $labels.namespace }}, method: {{ $labels.method }}, url: {{ $labels.url }}, HTTP 5xx error rate: {{ $value | humanizePercentage }}"

      - alert: TicketManagerSyncEngineersFailed
        expr: |
          sum by (namespace, service_name, job_name) (
            increase(
              ticket_manager_cron_job_sync_count_total{
                service_name="EngineerService",
                job_name="EngineerUserSync",
                result="failed"
              }[5m]
            )
          ) > 0
        labels:
          severity: major
          component: ticket-manager
          oncall_team: resilience
          response_class: ticket
        annotations:
          summary: "ticket-manager engineer synchronization failed"
          description: "namespace: {{ $labels.namespace }}, service: {{ $labels.service_name }}, job: {{ $labels.job_name }}, failures in the last 5 minutes: {{ $value | humanize }}"

      - alert: TicketManagerDispatchExecutionFailed
        expr: |
          (
            ticket_manager_dispatch_last_failed_execution_info{mode="live"} == 1
          )
          unless on (namespace, mode, execution_id)
          (
            ticket_manager_dispatch_last_failed_execution_info{mode="live"} offset 5m
          )
        labels:
          severity: major
          component: ticket-manager
          oncall_team: resilience
          response_class: ticket
        annotations:
          summary: "ticket-manager Dispatch Execution failed"
          description: "namespace: {{ $labels.namespace }}, execution ID: {{ $labels.execution_id }}, issue: {{ $labels.issue_key }}, failure code: {{ $labels.failure_code }}"