groups:
  - name: infra.rules
    rules:
      - alert: ManagerFailToGet
        expr: increase(infra_client_get_duration_seconds_count{job="manager",rlt="fail"}[5m]) > 10
        for: 10m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra Get from infra {{ $labels.name }} is failing.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerFailToGetIndex
        expr: increase(infra_client_get_index_duration_seconds_count{job="manager",rlt="fail"}[5m]) > 10
        for: 10m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra GetIndex from infra {{ $labels.name }} is failing.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerFailToPatch
        expr: increase(infra_client_patch_duration_seconds_count{job="manager",rlt="fail"}[1m]) > 30
        for: 30m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra Patch to infra {{ $labels.name }} is failing.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerFailToUpdate
        expr: increase(infra_client_update_duration_seconds_count{job="manager",rlt="fail"}[1m]) > 30
        for: 30m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra Update to infra {{ $labels.name }} is failing.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerFailToDelete
        expr: increase(infra_client_delete_duration_seconds_count{job="manager",rlt="fail"}[1m]) > 10
        for: 30m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra Delete to infra {{ $labels.name }} is failing.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerRedisXReadTooMany
        expr: increase(infra_consumer_redis_xread_total{job="manager",rlt="fail"}[1m]) > 10
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra redis stream {{ $labels.name }} call XRead too many.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerRedisResetLastIDTooMany
        expr: increase(infra_consumer_redis_reset_last_id_total{job="manager"}[10m]) > 5
        for: 30m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra redis stream {{ $labels.name }} reset lastID too many.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerReceiveMessageBlock
        expr: sum without (rlt) (increase(infra_consumer_infra_receive_message_total{job="manager",provider_type="aws-free-tier"}[1m])) == 0
        for: 10m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra ReceiveMessage from infra MQ {{ $labels.name }} is blocking.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerFailToReceiveMQ
        expr: increase(infra_consumer_infra_receive_message_total{job="manager",rlt="fail"}[5m]) > 10
        for: 5m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra ReceiveMessage from infra MQ {{ $labels.name }} is failing.
          kubernetes: '{{ $labels.cluster }}'
      - alert: ManagerReceiveDlqMQTooMany
        expr: increase(infra_consumer_infra_message_total{job="manager",from="dlq"}[1m]) > 30
        for: 20m
        labels:
          tier: serverless
          severity: critical
          response_class: page
          provider_type: aws-free-tier
          component: manager
          oncall_team: serverless
        annotations:
          message: infra infra DLQ MQ {{ $labels.name }} receive message too many.
          kubernetes: '{{ $labels.cluster }}'