groups:
  # Alerts group for VM cluster assumes that Grafana dashboard
  # https://grafana.com/grafana/dashboards/11176 is installed.
  # Please, update the `dashboard` annotation according to your setup.
  - name: vmalert
    interval: 30s
    rules:
    - alert: ConfigurationReloadFailure
      expr: vmalert_config_last_reload_successful != 1
      labels:
        oncall_team: o11y
        severity: warning
      annotations:
        message: "Configuration reload failed for vmalert instance {{ $labels.instance }}"
        description: "Configuration hot-reload failed for vmalert on instance {{ $labels.instance }}.
          Check vmalert's logs for detailed error message."

    - alert: AlertingRulesError
      expr: label_replace(sum(increase(vmalert_alerting_rules_errors_total{group!="vmAlertGroup"}[5m])) without(id), "affected_alertname", "$1", "alertname", "(.*)") > 0 # label_replace avoids alertname being overwrite by prometheus which cause PrometheusRuleFailures
      for: 30m
      labels:
        oncall_team: o11y
        severity: warning
      annotations:
        dashboard: "https://self-monitoring-grafana.observability.tidbcloud.com/d/LzldHAVnz?viewPanel=13&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
        message: "Alerting rules are failing for vmalert instance {{ $labels.instance }}"
        description: "Alerting rules execution is failing for \"{{ $labels.affected_alertname }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
          Check vmalert's logs for detailed error message."

    - alert: RecordingRulesError
      expr: sum(increase(vmalert_recording_rules_errors_total{group!="vmAlertGroup"}[5m])) without(id) > 0
      for: 30m
      labels:
        oncall_team: o11y
        severity: warning
      annotations:
        dashboard: "https://self-monitoring-grafana.observability.tidbcloud.com/d/LzldHAVnz?viewPanel=30&var-instance={{ $labels.instance }}&var-file={{ $labels.file }}&var-group={{ $labels.group }}"
        message: "Recording rules are failing for vmalert instance {{ $labels.instance }}"
        description: "Recording rules execution is failing for \"{{ $labels.recording }}\" from group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
          Check vmalert's logs for detailed error message."

    # - alert: TooManyMissedIterations
    #   expr: increase(vmalert_iteration_missed_total[5m]) > 0
    #   for: 15m
    #   labels:
    #     severity: warning
    #   annotations:
    #     message: "vmalert instance {{ $labels.instance }} is missing rules evaluations"
    #     description: "vmalert instance {{ $labels.instance }} is missing rules evaluations for group \"{{ $labels.group }}\" in file \"{{ $labels.file }}\".
    #       The group evaluation time takes longer than the configured evaluation interval. This may result in missed
    #       alerting notifications or recording rules samples. Try increasing evaluation interval or concurrency of
    #       group \"{{ $labels.group }}\". See https://docs.victoriametrics.com/victoriametrics/vmalert/#groups.
    #       If rule expressions are taking longer than expected, please see https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries."

    - alert: RemoteWriteErrors
      expr: increase(vmalert_remotewrite_errors_total[5m]) > 0
      for: 15m
      labels:
        oncall_team: o11y
        severity: warning
      annotations:
        message: "vmalert instance {{ $labels.instance }} is failing to push metrics to remote write URL"
        description: "vmalert instance {{ $labels.instance }} is failing to push metrics generated via alerting
          or recording rules to the configured remote write URL. Check vmalert's logs for detailed error message."

    - alert: RemoteWriteDroppingData
      expr: increase(vmalert_remotewrite_dropped_rows_total[5m]) > 0
      for: 5m
      labels:
        oncall_team: o11y
        severity: major
      annotations:
        message: "vmalert instance {{ $labels.instance }} is dropping data sent to remote write URL"
        description: "vmalert instance {{ $labels.instance }} is failing to send results of alerting or recording rules
          to the configured remote write URL. This may result into gaps in recording rules or alerts state.
          Check vmalert's logs for detailed error message."

    - alert: AlertmanagerErrors
      expr: increase(vmalert_alerts_send_errors_total[5m]) > 0
      for: 15m
      labels:
        oncall_team: o11y
        severity: warning
      annotations:
        message: "vmalert instance {{ $labels.instance }} is failing to send notifications to Alertmanager"
        description: "vmalert instance {{ $labels.instance }} is failing to send alert notifications to \"{{ $labels.addr }}\".
          Check vmalert's logs for detailed error message."