groups:
- name: vmagent
  rules:
  - alert: VMAgentCoreDNSLookupFailed
    expr: |
      sum(count_over_time({cluster_env="prod",container=~"victoria-metrics-agent|vmagent", namespace=~"monitoring|tidb.*",tenant_id!="1372813089187791289"}[2m] |~ "couldn't send.*lookup")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 5m
    labels:
      severity: critical
      response_class: page
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'VMAgent failed to send blocks due to dns lookup error (instance {{ $labels.instance }}). cluster: {{ $labels.cluster }}, cluster_env: {{ $labels.cluster_env }}, namespace: {{ $labels.namespace }}'
      summary: 'VMAgent fails to send blocks due to DNS lookup errors'
  - alert: VMAgentExceedsMaxScrapeSize
    expr: |
      sum(count_over_time({cluster_env="prod",container=~"victoria-metrics-agent|vmagent",cluster!~".*-f0[0-9]+"}[2m] |~ "exceeds -promscrape.maxScrapeSize")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'VMAgent scrape exceeds promscrape.maxScrapeSize or max_scrape_size in the scrape config (instance {{ $labels.instance }}). The vmagent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing scrape errors due to exceeding max scarpe size'
      summary: 'VMAgent scrape response exceeds the max scrape size limit'
  - alert: VMAgentExceedsMaxScrapeSizeServerless
    expr: |
      sum(count_over_time({cluster_env="prod",container=~"victoria-metrics-agent|vmagent",cluster=~".*-f0[0-9]+"}[2m] |~ "exceeds -promscrape.maxScrapeSize")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: vmagent
      oncall_team: o11y
      provider_type: aws-free-tier
      tier: serverless
    annotations:
      description: 'VMAgent scrape exceeds promscrape.maxScrapeSize or max_scrape_size in the scrape config (instance {{ $labels.instance }}). The vmagent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing scrape errors due to exceeding max scrape size (serverless cluster)'
      summary: 'VMAgent scrape response exceeds the max scrape size limit on serverless clusters'
  - alert: VMAgentBackoff
    expr: |
      sum(count_over_time({cluster_env="prod", namespace="monitoring", container="event-exporter"}[5m] |= `vmagent` | json | metadata_namespace=`monitoring` | reason=`BackOff`)) by (cluster_env, cluster, reason, message, metadata_name, tenant_id) > 0
    labels:
      severity: critical
      response_class: page
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The dataplane vmagent {{ $labels.cluster_env }} {{ $labels.cluster }} monitoring/{{ $labels.metadata_name }} is in status: {{ $labels.reason }}. message: {{ $labels.message }}'
      summary: 'Dataplane VMAgent is in backoff state'
  - alert: VMAgentRemoteWrite404NotFound
    expr: |
      sum(count_over_time({cluster_env="prod",namespace="monitoring",container="vmagent"}[5m] |= "404 Not Found")) by (cluster) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The dataplane vmagent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing 404 Not Found errors for 5 minutes'
  - alert: VectorSink403ForbiddenFor30m
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "response status: 403 Forbidden")) by (cluster_env, cluster) > 0
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing 403 Forbidden errors for 30 minutes'
  - alert: VectorAssumeRoleWithWebIdentityFailedOnce
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "Not authorized to perform sts:AssumeRoleWithWebIdentity")) by (cluster_env, cluster) > 0
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing assume role failure'
  - alert: VectorAssumeRoleWithWebIdentityFailed
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "Not authorized to perform sts:AssumeRoleWithWebIdentity")) by (cluster_env, cluster) > 0
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing assume role failures for 30 minutes'
  - alert: VectorAssumeRoleFailedOnce
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "is not authorized to perform: sts:AssumeRole on resource:")) by (cluster_env, cluster) > 0
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing assume role failure'
  - alert: VectorAssumeRoleFailed
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "is not authorized to perform: sts:AssumeRole on resource:")) by (cluster_env, cluster) > 0
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing assume role failures for 30 minutes'
  - alert: VectorNoOIDCProviderFoundFor5m
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "No OpenIDConnect provider found in your account")) by (cluster_env, cluster) > 0
    for: 5m
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing no oidc provider found errors for 5 minutes'
  - alert: VectorNoOIDCProviderFoundFor30m
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[5m] |= "No OpenIDConnect provider found in your account")) by (cluster_env, cluster) > 0
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing no oidc provider found errors 30 minutes'
  - alert: ClusterVectorSink403ForbiddenFor30m
    expr: |
      sum(count_over_time({cluster_env="prod",container="o11y-vector-agent",namespace=~"tidb.*",tenant_id!="1372813089187791289"}[5m] |= "response status: 403 Forbidden")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 30m
    labels:
      severity: critical
      response_class: page
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector cluster agent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing 403 Forbidden errors for 30 minutes'
  - alert: ClusterVectorAssumeRoleFailedOnce
    expr: |
      sum(count_over_time({cluster_env="prod",container="o11y-vector-agent",namespace=~"tidb.*",tenant_id!="1372813089187791289"}[5m] |= "is not authorized to perform: sts:AssumeRole on resource:")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector cluster agent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing assume role failure'
  - alert: ClusterVectorAssumeRoleFailed
    expr: |
      sum(count_over_time({cluster_env="prod",container="o11y-vector-agent",namespace=~"tidb.*",tenant_id!="1372813089187791289"}[5m] |= "is not authorized to perform: sts:AssumeRole on resource:")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector cluster agent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing assume role failures for 30 minutes'
  - alert: ClusterVectorNoOIDCProviderFoundFor5m
    expr: |
      sum(count_over_time({cluster_env="prod",container="o11y-vector-agent",namespace=~"tidb.*",tenant_id!="1372813089187791289"}[5m] |= "No OpenIDConnect provider found in your account")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 5m
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector cluster agent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing no oidc provider found errors for 5 minutes'
  - alert: ClusterVectorNoOIDCProviderFoundFor30m
    expr: |
      sum(count_over_time({cluster_env="prod",container="o11y-vector-agent",namespace=~"tidb.*",tenant_id!="1372813089187791289"}[5m] |= "No OpenIDConnect provider found in your account")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The vector cluster agent {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing no oidc provider found errors for 30 minutes'
  - alert: RotatedFilesDroppedDueToDiskUsage
    expr: |
      sum(count_over_time({cluster_env="prod",app="vector_log_agent"}[2m] |= "Rotated files dropped due to disk usage.")) by (cluster_env, cluster) > 0
    annotations:
      description: The vector log agent {{ $labels.cluster_env }} {{ $labels.cluster }} is experiencing rotated files dropped due to disk usage
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
  - alert: ClusterVectorDroppingEvents
    expr: |
      sum(count_over_time({cluster_env="prod",container="vector"}[5m] |= "dropping events")) by (cluster_env, cluster, namespace, tenant_id, instance) > 0
    for: 5m
    labels:
      severity: warning
      component: o11y
      oncall_team: o11y
    annotations:
      description: 'The cluster vector {{ $labels.cluster_env }} {{ $labels.cluster }} {{ $labels.namespace }} is experiencing dropping events'