groups: # Alerts group for VM cluster assumes that Grafana dashboard # https://grafana.com/grafana/dashboards/11176 is installed. # Please, update the `dashboard` annotation according to your setup. - name: vmcluster interval: 30s rules: - alert: VMclusterRequestTooManyErrorsToAPI expr: sum(increase(vm_http_request_errors_total[5m])) by (job, path) > 10 for: 10m labels: oncall_team: o11y severity: warning annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=52&var-job={{ $labels.job }}" summary: "Too many errors served for {{ $labels.job }} path {{ $labels.path }} " description: |- Requests to path {{ $labels.path }} are receiving {{ printf "%.2f" $value }}s errors in 5mins. Please verify if clients are sending correct requests. runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnLh0XmeJD9sEwXqb2BnStxb - alert: VMclusterRPCErrors expr: | ( sum(increase(vm_rpc_connection_errors_total[5m])) by(job, instance) + sum(increase(vm_rpc_dial_errors_total[5m])) by(job, instance) + sum(increase(vm_rpc_handshake_errors_total[5m])) by(job, instance) ) > 0 for: 15m labels: oncall_team: o11y severity: major annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=44&var-instance={{ $labels.instance }}" summary: "Too many RPC errors for {{ $labels.job }} (instance {{ $labels.instance }})" description: "RPC errors are interconnection errors between cluster components.\n Possible reasons for errors are misconfiguration, overload, network blips or unreachable components." runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnxsHGyuzs5T3oDWvJGrP7gc - alert: VMclusterConcurrentFlushesHitTheLimit expr: avg_over_time(vm_concurrent_addrows_current[1m]) >= vm_concurrent_addrows_capacity for: 15m labels: oncall_team: o11y severity: major annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=133&var-instance={{ $labels.instance }}" summary: "vmstorage on instance {{ $labels.instance }} is constantly hitting concurrent flushes limit" description: "The limit of concurrent flushes on instance {{ $labels.instance }} is equal to number of CPUs.\n When vmstorage constantly hits the limit it means that storage is overloaded and requires more CPU." runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnjqLT7zazAi2QW1MKsYJXAh - alert: VMclusterRowsRejectedOnIngestion expr: sum(rate(vm_rows_ignored_total[5m])) by (instance, reason) > 0 for: 15m labels: oncall_team: o11y severity: major annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=135&var-instance={{ $labels.instance }}" summary: "Some rows are rejected on \"{{ $labels.instance }}\" on ingestion attempt" description: "VM is rejecting to ingest rows on \"{{ $labels.instance }}\" due to the following reason: \"{{ $labels.reason }}\"" runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnzXHNvQxBhPbP5E2exLNIJD - alert: VMclusterTooHighChurnRate expr: | ( sum(rate(vm_new_timeseries_created_total[5m])) / sum(rate(vm_rows_inserted_total[5m])) ) > 0.1 for: 15m labels: oncall_team: o11y severity: warning annotations: dashboard: "http://localhost:3000/d/oS7Bi_0Wz?viewPanel=102" summary: "Churn rate is more than 10% for the last 15m" description: "VM constantly creates new time series.\n This effect is known as Churn Rate.\n High Churn Rate tightly connected with database performance and may result in unexpected OOM's or slow queries." - alert: VMclusterTooHighSlowInsertsRate expr: | ( sum(rate(vm_slow_row_inserts_total[2m])) / sum(rate(vm_rows_inserted_total[2m])) ) > 0.05 for: 30m labels: oncall_team: o11y severity: warning annotations: message: "Percentage of slow inserts is more than 5% for the last 30m" description: |- High rate of slow inserts may be a sign of resource exhaustion for the current load. It is likely more RAM is needed for optimal handling of the current number of active time series. For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }} runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnlj7hHUlgdZgDsWdVJRHVj9 - alert: VMclusterProcessNearFDLimits expr: (process_max_fds - process_open_fds) < 100 for: 5m labels: oncall_team: o11y severity: critical annotations: dashboard: 'http://self-monitoring-grafana.observability.tidbcloud.com/d/oS7Bi_0Wz?viewPanel=117&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}&var-instance={{ $labels.instance }}' summary: "Number of free file descriptors is less than 100 for \"{{ $labels.job }}\"(\"{{ $labels.instance }}\") for the last 5m" description: "Exhausting OS file descriptors limit can cause severe degradation of the process. Consider to increase the limit as fast as possible." runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn0mdwdaZC6nGSVpgIHhOhQX - alert: VMclusterLabelsLimitExceededOnIngestion expr: sum(increase(vm_metrics_with_dropped_labels_total[5m])) by (instance) > 0 for: 15m labels: oncall_team: o11y severity: warning annotations: dashboard: 'http://self-monitoring-grafana.observability.tidbcloud.com/d/oS7Bi_0Wz?viewPanel=116&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }}&var-instance={{ $labels.instance }}' message: "Metrics ingested to vminsert on {{ $labels.instance }} are exceeding labels limit" description: |- VictoriaMetrics limits the number of labels per each metric with `-maxLabelsPerTimeseries` command-line flag. This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured correctly or that clients which send these metrics aren't misbehaving. This prevents from ingesting metrics with too many labels. Please verify that `-maxLabelsPerTimeseries` is configured correctly or that clients which send these metrics aren't misbehaving. - alert: VMclusterVminsertVmstorageConnectionIsSaturated expr: rate(vm_rpc_send_duration_seconds_total[2m]) > 0.9 for: 15m labels: oncall_team: o11y severity: major annotations: message: "Connection between vminsert on {{ $labels.instance }} and vmstorage on {{ $labels.addr }} is saturated" description: |- The connection between vminsert (instance {{ $labels.instance }}) and vmstorage (instance {{ $labels.addr }}) is saturated by more than 90% and vminsert won't be able to keep up.\n This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase the total number of vminsert -> vmstorage links. For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }} - alert: VMClusterMemoryUsageTooHigh expr: (process_resident_memory_anon_bytes / vm_available_memory_bytes) > 0.95 for: 10m labels: oncall_team: o11y severity: critical annotations: message: |- More than 95% of memory used by {{ $labels.job }} ({{ $labels.instance }}) ({{ $labels.pod }}) during the last 30m. current value: {{ printf "%.2f" $value }} description: |- Too high memory usage may result into multiple issues such as OOMs or degraded performance. For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }} runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn5OyPqV4usCspdQXQY321Dh - alert: VMClusterMemoryUsageTooHigh expr: (process_resident_memory_anon_bytes / vm_available_memory_bytes) > 0.90 for: 30m labels: oncall_team: o11y severity: major annotations: message: |- More than 90% of memory used by {{ $labels.job }} ({{ $labels.instance }}) ({{ $labels.pod }}) during the last 30m. current value: {{ printf "%.2f" $value }} description: |- Too high memory usage may result into multiple issues such as OOMs or degraded performance. For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }} runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcn5OyPqV4usCspdQXQY321Dh - alert: VMClusterCPUUsageTooHigh annotations: description: |- Too high CPU usage may be a sign of insufficient resources and make process unstable. For scaling guides, please Refer to the dashboard https://self-monitoring-grafana.observability.tidbcloud.com/d/0Wz_just1900/victoriametrics-capacity-planning?orgId=1&viewPanel=149&var-ds=Prometheus-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll ".*regional-(.+)" "$1" }} message: |- More than 90% of CPU is used by {{ $labels.job }}/{{ $labels.pod }} during last 30m. current value: {{ printf "%.2f" $value }} runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcndy0CYHYi4tcJPMpWeqWqCf expr: rate(process_cpu_seconds_total{job=~"vmselect.*|vminsert.*|vmstorage.*"}[5m]) / process_cpu_cores_available > 0.9 for: 30m labels: oncall_team: o11y severity: major - alert: VMClusterCPUThrottlingHigh annotations: description: throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.pod }}/{{ $labels.container }}, this might result in performance issue of vmcluster. message: container {{ $labels.pod }}/{{ $labels.container }} experience elevated CPU throttling. runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcniKUi42cYGwQESW3nBkcrqb expr: |- sum(increase(container_cpu_cfs_throttled_periods_total{namespace="observability",container=~"vmselect.*|vminsert.*|vmstorage.*"}[5m])) by (container, pod, namespace) / sum(increase(container_cpu_cfs_periods_total{namespace="observability",container=~"vmselect.*|vminsert.*|vmstorage.*"}[5m])) by (container, pod, namespace) > 0.5 for: 30m labels: oncall_team: o11y severity: major - alert: VMClusterTooHighQueryLoad annotations: description: 'Instance {{`{{`}} $labels.instance {{`}}`}} ({{`{{`}} $labels.job {{`}}`}}) is failing to serve read queries during last 15m. Concurrency limit `-search.maxConcurrentRequests` was reached on this instance and extra queries were put into the queue for `-search.maxQueueDuration` interval. But even after waiting in the queue these queries weren''t served. This happens if instance is overloaded with the current workload, or datasource is too slow to respond. Possible solutions are the following: * reduce the query load; * increase compute resources or number of replicas; * adjust limits `-search.maxConcurrentRequests` and `-search.maxQueueDuration`. See more at https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries. ' message: 'Read queries fail with timeout for {{ $labels.job }} on instance {{ $labels.instance }}' expr: increase(vm_concurrent_select_limit_timeout_total[5m]) > 0 for: 15m labels: oncall_team: o11y severity: major - alert: VMClusterQueryLatencyTooHigh expr: max(vm_request_duration_seconds{quantile="0.99", path=~"/api/v1/query|/api/v1/query_range"}) by (path) > 4 for: 10m labels: oncall_team: o11y severity: major annotations: message: "p99 latency of {{ $labels.path }} requests has been over 4s for the last 10m (current value: {{ printf \"%.2f\" $value }}s)" description: |- Read queries served by vmselect are consistently slow. This may be caused by heavy queries, high churn rate or insufficient resources. See more at https://docs.victoriametrics.com/victoriametrics/troubleshooting/#slow-queries.