apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: metrics-integration-alert
spec:
  groups:
    # Alerts group for VM cluster assumes that Grafana dashboard
    # https://grafana.com/grafana/dashboards/11176 is installed.
    # Please, update the `dashboard` annotation according to your setup.
    - name: metrics-pipeline-alerts
      interval: 30s
      rules:
      - alert: MetricsPipelineTsLagMoreThan1Min
        expr: (metrics_integration_pipeline_upstream_timestamp{} - on (cluster_id) metrics_integration_pipeline_downstream_timestamp{}) > 60
        for: 5m
        labels:
          severity: major
          component: metrics-integration
          oncall_team: resilience
        annotations:
          component: metrics-integration
          message: "cluster: {{ $labels.cluster_id }}"
          description: "pipeline lag more than 1min, please take a look"
      - alert: MetricsPipelineDownstreamMetricsMissing
        expr: metrics_integration_pipeline_upstream_timestamp{} unless on (cluster_id) metrics_integration_pipeline_downstream_timestamp{} > 0
        for: 5m
        labels:
          severity: major
          component: metrics-integration
          oncall_team: resilience
        annotations:
          component: metrics-integration
          message: "cluster: {{ $labels.cluster_id }}"
          description: "pipeline downstream ts missing, please check"
      - alert: MetricsPipelineDownstreamTsBehindNowMoreThan1Min
        expr: |
          (
            (time() - max by (cluster_id) (metrics_integration_pipeline_downstream_timestamp{})) > 60
            and
            max by (cluster_id) (metrics_integration_pipeline_downstream_timestamp{}) > 0
          )
          and on (cluster_id)
          max by (cluster_id) (
            metrics_integration_pipeline_upstream_timestamp{} > 0
          )
        for: 5m
        labels:
          severity: major
          component: metrics-integration
          oncall_team: resilience
        annotations:
          component: metrics-integration
          message: "cluster: {{ $labels.cluster_id }}"
          description: "downstream ts is behind current time by more than 1 minute for 5m"