groups:
  - name: vm-operator
    interval: 30s
    rules:
    - alert: ReconcileErrors
      annotations:
        description: 'Operator cannot parse response from k8s api server when updating {{ $labels.controller }}, possible bug: {{ $value }}, check operator logs'
        message: 'Too many errors at reconcile loop of operator when updating {{ $labels.controller }}: {{ $value }}'
      expr: |-
        sum(
          rate(
            controller_runtime_reconcile_errors_total{
              job=~".*((victoria.*)|vm)-?operator"
            }[5m]
          )
        ) by (controller) > 0
      for: 10m
      labels:
        oncall_team: o11y
        severity: warning
    - alert: HighQueueDepth
      annotations:
        description: 'Operator cannot handle reconciliation load for controller: {{ $labels.name }}, current depth: {{ $value }}'
        message: 'Too many {{ $labels.name }} in queue: {{ $value }}'
      expr: |-
        sum(
          workqueue_depth{
            job=~".*((victoria.*)|vm)-?operator",
            name=~"(vmagent|vmalert|vmalertmanager|vmauth|vmcluster|vmnodescrape|vmpodscrape|vmprobe|vmrule|vmservicescrape|vmsingle|vmstaticscrape)"
          }
        ) by (name,job) > 10
      for: 15m
      labels:
        oncall_team: o11y
        severity: warning