apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: self-monitoring-o11y-vmcluster
  namespace: {{ .Values.namespace.selfMonitoring }}
spec:
  groups:
    # Alerts group for VM cluster assumes that Grafana dashboard
    # https://grafana.com/grafana/dashboards/11176 is installed.
    # Please, update the `dashboard` annotation according to your setup.
    - name: vmcluster
      interval: 30s
      rules:
        - alert: VMclusterDiskRunsOutOfSpaceIn3Days
          expr: |
            vm_free_disk_space_bytes / ignoring(path)
            (
              (
                rate(vm_rows_added_to_storage_total[1d]) -
                ignoring(type) rate(vm_deduplicated_samples_total{type="merge"}[1d])
              )
              * scalar(
                sum(vm_data_size_bytes{type!="indexdb"}) /
                sum(vm_rows{type!="indexdb"})
              )
            ) < 3 * 24 * 3600 > 0
          for: 30m
          labels:
            severity: critical
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=113&var-instance={{`{{ $labels.instance }}`}}"
            summary: "Instance {{`{{ $labels.instance }}`}} will run out of disk space in 3 days"
            description: "Taking into account current ingestion rate, free disk space will be enough only
                for {{`{{ $value | humanizeDuration }}`}} on instance {{`{{ $labels.instance }}`}}.\n
                Consider to limit the ingestion rate, decrease retention or scale the disk space up if possible."
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnQJanbQ33wrth9xv9ij1rq1
        # Use KubePersistentVolumeFillingUp instead
        # - alert: VMclusterDiskRunsOutOfSpace
        #   expr: |
        #     sum(vm_data_size_bytes) by(instance) /
        #     (
        #     sum(vm_free_disk_space_bytes) by(instance) +
        #     sum(vm_data_size_bytes) by(instance)
        #     ) > 0.8
        #   for: 30m
        #   labels:
        #     severity: critical
        #   annotations:
        #     dashboard: http://localhost:3000/d/oS7Bi_0Wz?viewPanel=110&var-instance={{`{{ $labels.instance }}`}}"
        #     summary: "Instance {{`{{ $labels.instance }}`}} will run out of disk space soon"
        #     description: "Disk utilisation on instance {{`{{ $labels.instance }}`}} is more than 80%.\n
        #       Having less than 20% of free disk space could cripple merges processes and overall performance.
        #       Consider to limit the ingestion rate, decrease retention or scale the disk space if possible."
        - alert: VMclusterRequestErrorsToAPI
          expr: increase(vm_http_request_errors_total[5m]) > 0
          for: 15m
          labels:
            severity: warning
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=52&var-instance={{`{{ $labels.instance }}`}}"
            summary: "Too many errors served for {{`{{ $labels.job }}`}} path {{`{{ $labels.path }}`}} (instance {{`{{ $labels.instance }}`}})"
            description: |-
              Requests to path {{`{{ $labels.path }}`}} are receiving {{`{{ printf "%.2f" $value }}`}}s errors in 5mins.
              Please verify if clients are sending correct requests.
        - alert: VMclusterRequestTooManyErrorsToAPI
          expr: sum(increase(vm_http_request_errors_total[5m])) by (job, path) > 10
          for: 10m
          labels:
            severity: major
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=52&var-job={{`{{ $labels.job }}`}}"
            summary: "Too many errors served for {{`{{ $labels.job }}`}} path {{`{{ $labels.path }}`}} "
            description: |-
              Requests to path {{`{{ $labels.path }}`}} are receiving {{`{{ printf "%.2f" $value }}`}}s errors in 5mins.
              Please verify if clients are sending correct requests.
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnLh0XmeJD9sEwXqb2BnStxb
        - alert: VMclusterRPCErrors
          expr: |
            (
            sum(increase(vm_rpc_connection_errors_total[5m])) by(job, instance)
            +
            sum(increase(vm_rpc_dial_errors_total[5m])) by(job, instance)
            +
            sum(increase(vm_rpc_handshake_errors_total[5m])) by(job, instance)
            ) > 0
          for: 15m
          labels:
            severity: major
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=44&var-instance={{`{{ $labels.instance }}`}}"
            summary: "Too many RPC errors for {{`{{ $labels.job }}`}} (instance {{`{{ $labels.instance }}`}})"
            description: "RPC errors are interconnection errors between cluster components.\n
              Possible reasons for errors are misconfiguration, overload, network blips or unreachable components."
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnxsHGyuzs5T3oDWvJGrP7gc

        - alert: VMclusterConcurrentFlushesHitTheLimit
          expr: avg_over_time(vm_concurrent_addrows_current[1m]) >= vm_concurrent_addrows_capacity
          for: 15m
          labels:
            severity: major
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=133&var-instance={{`{{ $labels.instance }}`}}"
            summary: "vmstorage on instance {{`{{ $labels.instance }}`}} is constantly hitting concurrent flushes limit"
            description: "The limit of concurrent flushes on instance {{`{{ $labels.instance }}`}} is equal to number of CPUs.\n
              When vmstorage constantly hits the limit it means that storage is overloaded and requires more CPU."
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnjqLT7zazAi2QW1MKsYJXAh

        - alert: VMclusterTooManyLogs
          expr: sum(increase(vm_log_messages_total{level="error"}[5m])) by (job, instance) > 0
          for: 15m
          labels:
            severity: warning
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=104&var-instance={{`{{ $labels.instance }}`}}"
            summary: "Too many logs printed for job \"{{`{{ $labels.job }}`}}\" ({{`{{ $labels.instance }}`}})"
            description: "Logging rate for job \"{{`{{ $labels.job }}`}}\" ({{`{{ $labels.instance }}`}}) is {{`{{ $value }}`}} for last 15m.\n
              Worth to check logs for specific error messages."

        - alert: VMclusterRowsRejectedOnIngestion
          expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0
          for: 15m
          labels:
            severity: major
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=135&var-instance={{`{{ $labels.instance }}`}}"
            summary: "Some rows are rejected on \"{{`{{ $labels.instance }}`}}\" on ingestion attempt"
            description: "VM is rejecting to ingest rows on \"{{`{{ $labels.instance }}`}}\" due to the
              following reason: \"{{`{{ $labels.reason }}`}}\""
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnzXHNvQxBhPbP5E2exLNIJD

        - alert: VMclusterTooHighChurnRate
          expr: |
            (
              sum(rate(vm_new_timeseries_created_total[5m]))
              /
              sum(rate(vm_rows_inserted_total[5m]))
            ) > 0.1
          for: 15m
          labels:
            severity: warning
          annotations:
            dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=102"
            summary: "Churn rate is more than 10% for the last 15m"
            description: "VM constantly creates new time series.\n
              This effect is known as Churn Rate.\n
              High Churn Rate tightly connected with database performance and may
              result in unexpected OOM's or slow queries."


        - alert: VMclusterTooHighSlowInsertsRate
          expr: |
            (
              sum(rate(vm_slow_row_inserts_total[5m]))
              /
              sum(rate(vm_rows_inserted_total[5m]))
            ) > 0.05
          for: 15m
          labels:
            severity: major
          annotations:
            message: "Percentage of slow inserts is more than 5% for the last 15m"
            description: |-
              High rate of slow inserts may be a sign of resource exhaustion
              for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series.
              For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{`{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}`}}
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnlj7hHUlgdZgDsWdVJRHVj9

        - alert: VMclusterProcessNearFDLimits
          expr: (process_max_fds - process_open_fds) < 100
          for: 5m
          labels:
            severity: critical
          annotations:
            dashboard: 'http://self-monitoring-grafana.observability.tidbcloud.com/d/oS7Bi_0Wz?viewPanel=117&var-ds=Prometheus-regional-{{`{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}`}}&var-instance={{`{{ $labels.instance }}`}}'
            summary: "Number of free file descriptors is less than 100 for \"{{`{{ $labels.job }}`}}\"(\"{{`{{ $labels.instance }}`}}\") for the last 5m"
            description: "Exhausting OS file descriptors limit can cause severe degradation of the process.
            Consider to increase the limit as fast as possible."
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn0mdwdaZC6nGSVpgIHhOhQX

        - alert: VMclusterLabelsLimitExceededOnIngestion
          expr: sum(increase(vm_metrics_with_dropped_labels_total[5m])) by (instance) > 0
          for: 15m
          labels:
            severity: warning
          annotations:
            dashboard: 'http://self-monitoring-grafana.observability.tidbcloud.com/d/oS7Bi_0Wz?viewPanel=116&var-ds=Prometheus-regional-{{`{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}`}}&var-instance={{`{{ $labels.instance }}`}}'
            message: "Metrics ingested to vminsert on {{`{{ $labels.instance }}`}} are exceeding labels limit"
            description: "VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag.\n
            This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured
            correctly or that clients which send these metrics aren't misbehaving."

        - alert: VMclusterVminsertVmstorageConnectionIsSaturated
          expr: rate(vm_rpc_send_duration_seconds_total[5m]) > 0.9
          for: 15m
          labels:
            severity: warning
          annotations:
            message: "Connection between vminsert on {{`{{ $labels.instance }}`}} and vmstorage on {{`{{ $labels.addr }}`}} is saturated"
            description: |-
              The connection between vminsert (instance {{`{{ $labels.instance }}`}}) and vmstorage (instance {{`{{ $labels.addr }}`}})
              is saturated by more than 90% and vminsert won't be able to keep up.\n
              This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase
              the total number of vminsert -> vmstorage links.
              For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{`{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}`}}

        - alert: VMClusterMemoryUsageTooHigh
          expr: (process_resident_memory_anon_bytes / vm_available_memory_bytes) > 0.85
          for: 10m
          labels:
            severity: major
          annotations:
            message: |-
              More than 85% of memory used by {{`{{ $labels.job }}`}} ({{`{{ $labels.instance }}`}}) ({{`{{ $labels.pod }}`}}) during the last 10m.
              current value: {{`{{ printf "%.2f" $value }}`}}
            description: |-
              Too high memory usage may result into multiple issues such as OOMs or degraded performance.
              For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{`{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}`}}
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn5OyPqV4usCspdQXQY321Dh
        - alert: VMClusterCPUUsageTooHigh
          annotations:
            description: |-
              Too high CPU usage may be a sign of insufficient resources and make process unstable.
              For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{`{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}`}}
            message: |-
              More than 90% of CPU is used by {{`{{ $labels.job }}`}}/{{`{{ $labels.pod }}`}} during last 30m.
              current value: {{`{{ printf "%.2f" $value }}`}}
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcndy0CYHYi4tcJPMpWeqWqCf
          expr: rate(process_cpu_seconds_total{job=~"vmselect.*|vminsert.*|vmstorage.*"}[5m]) / process_cpu_cores_available > 0.9
          for: 30m
          labels:
            severity: major
        - alert: VMClusterCPUThrottlingHigh
          annotations:
            description: throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for container {{`{{ $labels.pod }}`}}/{{`{{ $labels.container }}`}}, this might result in performance issue of vmcluster.
            message: container {{`{{ $labels.pod }}`}}/{{`{{ $labels.container }}`}} experience elevated CPU throttling.
            runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcniKUi42cYGwQESW3nBkcrqb
          expr: |-
            sum(increase(container_cpu_cfs_throttled_periods_total{namespace="observability",container=~"vmselect.*|vminsert.*|vmstorage.*"}[5m])) by (container, pod, namespace)
              /
            sum(increase(container_cpu_cfs_periods_total{namespace="observability",container=~"vmselect.*|vminsert.*|vmstorage.*"}[5m])) by (container, pod, namespace)
              > 0.5
          for: 30m
          labels:
            severity: major