{ "__inputs": [ { "name": "DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1", "label": "Prometheus-regional-ap-northeast-1", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" }, { "name": "DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1-SEED", "label": "Prometheus-regional-ap-northeast-1-seed", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" }, { "name": "DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1-SERVERLESS", "label": "Prometheus-regional-ap-northeast-1-serverless", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": [], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.0.1" }, { "type": "panel", "id": "piechart", "name": "Pie chart", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "text", "name": "Text", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "datasource", "uid": "grafana" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "enable": true, "expr": "sum(ALERTS{alertgroup=\"vmcluster\",alertstate=\"firing\",show_at=\"dashboard\"}) by(alertname)", "iconColor": "red", "name": "alerts", "titleFormat": "{{alertname}}" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "enable": true, "expr": "sum(vm_app_version{job=~\"$job\"}) by(short_version) unless (sum(vm_app_version{job=~\"$job\"} offset 20m) by(short_version))", "hide": true, "iconColor": "dark-blue", "name": "version change", "textFormat": "{{short_version}}", "titleFormat": "Version change" } ] }, "description": "Overview for cluster VictoriaMetrics v1.86.0 or higher", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "iteration": 1751536542138, "links": [ { "icon": "doc", "tags": [], "targetBlank": true, "title": "Cluster Wiki", "type": "link", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/wiki/Cluster-VictoriaMetrics" }, { "icon": "external link", "tags": [], "targetBlank": true, "title": "Found a bug?", "type": "link", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues" }, { "icon": "external link", "tags": [], "targetBlank": true, "title": "New releases", "type": "link", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/releases" } ], "liveNow": false, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 204, "panels": [], "title": "VM Cluster Topology", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false, "minWidth": 50 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "displayName", "value": "Count" } ] }, { "matcher": { "id": "byName", "options": "Expected Count" }, "properties": [ { "id": "custom.width", "value": 172 }, { "id": "custom.displayMode", "value": "color-background-solid" }, { "id": "color", "value": { "fixedColor": "red", "mode": "thresholds" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "transparent", "value": null }, { "color": "semi-dark-red", "value": 1 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 1 }, "id": 149, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [] }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(vm_app_version{job=~\".*-vmcluster\"}) by(job, short_version)", "format": "table", "instant": true, "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "(sum(vm_app_version{job=~\"vmstorage.*\"}) by (job) + 2)\nand ignoring(job)\n(\n sum(rate(vm_slow_row_inserts_total[5m]))\n /\n sum(rate(vm_rows_inserted_total[5m]))\n) > 0.02", "format": "table", "hide": false, "instant": true, "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(vm_app_version{job=~\"vminsert.*\", instance=~\"$instance\"}) by(job) + 2", "format": "table", "hide": true, "instant": true, "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(vm_app_version{job=~\"vmselect.*\", instance=~\"$instance\"}) by(job) + 2", "format": "table", "hide": true, "instant": true, "range": false, "refId": "D" } ], "transformations": [ { "id": "seriesToColumns", "options": { "byField": "job" } }, { "id": "organize", "options": { "excludeByName": { "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true }, "indexByName": {}, "renameByName": { "Value #A": "Current Count", "Value #B": "Expected Count", "Value #C": "Expected Count", "Value #D": "Expected Count" } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 1 }, "id": 62, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(min_over_time(up{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by (job)", "format": "time_series", "instant": false, "legendFormat": "{{job}}", "refId": "A" } ], "title": "Uptime ($job)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Value" }, "properties": [ { "id": "custom.hidden", "value": true } ] }, { "matcher": { "id": "byName", "options": "Time" }, "properties": [ { "id": "custom.hidden", "value": true } ] } ] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 6 }, "id": 183, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "job" } ] }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(flag{is_set=\"true\", job=~\"$job\", instance=~\"$instance\"}) by(job, instance, name, value)", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "title": "Non-default flags", "transformations": [ { "id": "groupBy", "options": { "fields": { "instance": { "aggregations": [ "uniqueValues" ], "operation": "aggregate" }, "job": { "aggregations": [], "operation": "groupby" }, "name": { "aggregations": [], "operation": "groupby" }, "value": { "aggregations": [], "operation": "groupby" } } } } ], "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 12, "y": 6 }, "id": 220, "interval": "1m", "links": [], "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "(sum(kube_pod_container_resource_requests{pod=~\"^vmalert.*\", resource=\"cpu\"}) or vector(0)) \n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "vmalert", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace!~\"self-monitoring|observability\", pod=~\"^prometheus.*\", resource=\"cpu\"}) or vector(0))\n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "prometheus", "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace=\"observability\", pod=~\"^vm.*\", resource=\"cpu\"}) or vector(0)) \n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "vmcluster", "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace=\"kube-system\", resource=\"cpu\"}) or vector(0)) \n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "kube-system", "range": false, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace=\"self-monitoring\", resource=\"cpu\"}) or vector(0)) \n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "self-monitoring", "range": false, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(namespace_cpu:kube_pod_container_resource_requests:sum{}) - (sum(kube_pod_container_resource_requests{pod=~\".*(loki|vm).*\", resource=\"cpu\"}) or vector(0)) - (sum(kube_pod_container_resource_requests{namespace!~\"self-monitoring|observability\", pod=~\"^prometheus.*\", resource=\"cpu\"}) or vector(0)) - (sum(kube_pod_container_resource_requests{namespace=\"kube-system\", resource=\"cpu\"}) or vector(0)) - (sum(kube_pod_container_resource_requests{namespace=\"self-monitoring\", resource=\"cpu\"}) or vector(0)))\n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "others", "range": false, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace=\"observability\", pod=~\".*loki.*\", resource=\"cpu\"}) or vector(0)) \n/\nsum(namespace_cpu:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "Loki", "range": false, "refId": "G" } ], "title": "CPU Requests Commitment (%)", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 6 }, "id": 222, "interval": "1m", "links": [], "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace!~\"self-monitoring|observability\", pod=~\"^prometheus.*\", resource=\"memory\"}) or vector(0)) \n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "prometheus", "range": false, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace=\"observability\",pod=~\".*vm.*\", resource=\"memory\"}) or vector(0)) \n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "vmcluster", "range": false, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{namespace=\"kube-system\"})\n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "kube-system", "range": false, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "sum(namespace_memory:kube_pod_container_resource_requests:sum{namespace=\"self-monitoring\"})\n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "self-monitoring", "range": false, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(namespace_memory:kube_pod_container_resource_requests:sum)-(sum(kube_pod_container_resource_requests{pod=~\".*(vm|loki).*\", resource=\"memory\"}) or vector(0))-(sum(kube_pod_container_resource_requests{namespace!~\"self-monitoring|observability\", pod=~\"^prometheus.*\", resource=\"memory\"}) or vector(0)) - (sum(kube_pod_container_resource_requests{namespace=\"kube-system\", resource=\"memory\"}) or vector(0)) - (sum(kube_pod_container_resource_requests{namespace=\"self-monitoring\", resource=\"memory\"}) or vector(0))) \n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "Others", "range": false, "refId": "F" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace=\"observability\", pod=~\".*loki.*\", resource=\"memory\"}) or vector(0)) \n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "Loki", "range": false, "refId": "G" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "(sum(kube_pod_container_resource_requests{namespace!=\"observability\",pod=~\".*vmalert.*\", resource=\"memory\"}) or vector(0)) \n/\nsum(namespace_memory:kube_pod_container_resource_requests:sum{})", "hide": false, "instant": true, "legendFormat": "vmalert", "range": false, "refId": "A" } ], "title": "Memory Requests Commitment (%)", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 6 }, "id": 224, "interval": "1m", "links": [], "options": { "legend": { "displayMode": "table", "placement": "right", "values": [ "percent" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "(sum(kubelet_volume_stats_capacity_bytes{namespace!~\"self-monitoring|observability\", persistentvolumeclaim=~\"^prometheus.*\"}) or vector(0)) / sum(kubelet_volume_stats_capacity_bytes)", "hide": false, "legendFormat": "prometheus", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "(sum(kubelet_volume_stats_capacity_bytes{namespace=\"observability\", persistentvolumeclaim=~\"^vm.*\"}) or vector(0)) / sum(kubelet_volume_stats_capacity_bytes)", "hide": false, "legendFormat": "vmcluster", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "(sum(kubelet_volume_stats_capacity_bytes{namespace=\"self-monitoring\"}) or vector(0)) / sum(kubelet_volume_stats_capacity_bytes)", "hide": false, "legendFormat": "self-monitoring", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "(sum(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\".*loki.*\"}) or vector(0)) / sum(kubelet_volume_stats_capacity_bytes)", "hide": false, "legendFormat": "loki", "range": true, "refId": "E" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "(((sum(kubelet_volume_stats_capacity_bytes{}) or vector(0))) - ((sum(kubelet_volume_stats_capacity_bytes{namespace=\"observability\", persistentvolumeclaim=~\"^vm.*\"}) or vector(0))) - ((sum(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\".*loki.*\"}) or vector(0))) - ((sum(kubelet_volume_stats_capacity_bytes{namespace!~\"self-monitoring|observability\", persistentvolumeclaim=~\"^prometheus.*\"}) or vector(0)))-(sum(kubelet_volume_stats_capacity_bytes{namespace=\"self-monitoring\"}) or vector(0)))/sum(kubelet_volume_stats_capacity_bytes)", "hide": false, "legendFormat": "others", "range": true, "refId": "A" } ], "title": "Persistent Volume Claim (%)", "type": "piechart" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 209, "panels": [], "title": "Biz Cluster Stats", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 14 }, "id": 213, "interval": "1m", "links": [], "options": { "displayLabels": [ "value" ], "legend": { "displayMode": "table", "placement": "right", "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "sum(observability_cluster_info{}) by(status)", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A" } ], "title": "All Cluster Count", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 14 }, "id": 212, "interval": "1m", "links": [], "options": { "displayLabels": [ "value" ], "legend": { "displayMode": "table", "placement": "right", "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "sum(observability_cluster_info{bizType=~\"tidbcloud|dev-tidbcloud|staging-tidbcloud\", applicationID=~\"^[0-9].*[0-9]$\"}) by(status)", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A" } ], "title": "Dedicated Cluster Count", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1-SEED}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "id": 207, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1-SEED}" }, "expr": "count(last_over_time(up{job=\"node-exporter\"}[5m])) * absent(up{job=\"self-monitoring-grafana\"})", "refId": "A" } ], "title": "Dedicated Nodes Count(Prod Only)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [ { "options": { "PAUSED": { "color": "yellow", "index": 0 } }, "type": "value" } ], "noValue": "0", "unit": "none" }, "overrides": [ { "matcher": { "id": "byName", "options": "PAUSED" }, "properties": [ { "id": "color", "value": { "fixedColor": "#FADE2A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 21 }, "id": 215, "interval": "1m", "links": [], "options": { "displayLabels": [ "value" ], "legend": { "displayMode": "table", "placement": "right", "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "sum(observability_cluster_info{applicationID=\"datamigration\"}) by(status)", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A" } ], "title": "DM Clusters Count", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [], "noValue": "0", "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 21 }, "id": 214, "interval": "1m", "links": [], "options": { "displayLabels": [ "value" ], "legend": { "displayMode": "table", "placement": "right", "values": [ "value" ] }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "sum(observability_cluster_info{bizType=~\"prod-devtier|staging-devtier|dev-devtier\", applicationID=~\"^[0-9].*[0-9]$\"}) by (status)", "format": "time_series", "instant": true, "intervalFactor": 2, "legendFormat": "{{status}}", "refId": "A" } ], "title": "Serverless Cluster Count", "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1-SERVERLESS}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "id": 216, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1-SERVERLESS}" }, "expr": "count(last_over_time(up{job=\"node-exporter\"}[5m])) * absent(up{job=\"self-monitoring-grafana\"})", "refId": "A" } ], "title": "Serverless Nodes Count(Prod Only)", "type": "timeseries" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$ds" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, "id": 137, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "How many datapoints are in storage", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 11 }, "id": 131, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Total datapoints", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total amount of disk space claimed", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 6, "y": 11 }, "id": 35, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": false, "expr": "sum(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"vmstorage.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Disk space capacity", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total amount of used disk space / Total volume claimed", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 9, "y": 11 }, "id": 237, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": false, "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\"})\n/\nsum(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"vmstorage.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Disk space usage(%)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate of HTTP read requests.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "req/s" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 11 }, "id": 130, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path=~\"/select/.*\"}[$__rate_interval]))", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Read requests", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total number of available CPUs for all VM components. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 18, "y": 11 }, "id": 126, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Available CPU", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total number of available CPUs for all VM components. ", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 21, "y": 11 }, "id": 235, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{resource=\"cpu\",namespace=\"observability\",pod=~\"$job.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Request CPU", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of active time series with new data points inserted during the last hour. High value may result in ingestion slowdown. \n\nSee more details here https://docs.victoriametrics.com/FAQ.html#what-is-an-active-time-series", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 0, "y": 14 }, "id": 34, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"}[1h]))", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Active series", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the datapoints ingestion rate, including replication factor.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 6, "y": 14 }, "id": 124, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(vm_vminsert_metrics_read_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])) ", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "range": false, "refId": "A" } ], "title": "Ingestion rate", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Average disk usage per datapoint.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 6, "x": 12, "y": 14 }, "id": 112, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(vm_data_size_bytes{job=~\"$job_storage\", type!~\"indexdb.*\"}) / sum(vm_rows{job=~\"$job_storage\", type!~\"indexdb.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Bytes per point", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total size of available memory for all VM components.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 18, "y": 14 }, "id": 128, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Available memory", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Total size of available memory for all VM components.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 21, "y": 14 }, "id": 236, "links": [], "maxDataPoints": 100, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(kube_pod_container_resource_requests{resource=\"memory\",pod=~\"$job.*\"})", "format": "time_series", "instant": true, "interval": "", "intervalFactor": 1, "legendFormat": "", "refId": "A" } ], "title": "Request memory", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Percentage of used RSS memory (resident).\nThe RSS memory shows the amount of memory recently accessed by the application. It includes anonymous memory and data from recently accessed files (aka page cache).\nThe application's performance will significantly degrade when memory usage is close to 100%.\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "WSS / Requested" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "unit", "value": "percentunit" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, "id": 232, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(process_resident_memory_bytes{job=~\"$job\"}) ", "hide": false, "interval": "", "legendFormat": "RSS usage", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(container_memory_working_set_bytes{namespace=\"observability\",image!=\"\",pod=~\"$job.*\"}) ", "hide": false, "legendFormat": "WSS usage", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_cache_size_bytes{job=~\"$job\",type=~\"indexdb/.*\"})", "hide": false, "legendFormat": "indexdb Cache Size", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(container_memory_working_set_bytes{namespace=\"observability\",image!=\"\",pod=~\"$job.*\"})\n/\nsum(kube_pod_container_resource_requests{resource=\"memory\",pod=~\"$job.*\"})\n ", "hide": false, "legendFormat": "WSS / Requested", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_cache_size_bytes{job=~\"$job\",type=~\"storage.*\"})", "hide": false, "legendFormat": "storage Cache Size", "range": true, "refId": "E" } ], "title": "Memory Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.\nSafe memory usage % considered to be below 80%\n\nClick on the line and choose Drilldown to show memory usage per instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, "id": 234, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RSS anonymous memory % usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Divide CPU usage by CPU request.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 25 }, "id": 228, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\nsum(rate(container_cpu_usage_seconds_total{pod=~\".*vmstorage.*\",image!=\"\"}[5m])) by (pod)\n/ \nsum(kube_pod_container_resource_requests{resource=\"cpu\",pod=~\".*vmstorage.*\"}) by (pod)\n)\n", "hide": false, "legendFormat": "max usage", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(container_cpu_usage_seconds_total{pod=~\".*vmstorage.*\",image!=\"\"}[$__rate_interval]))\n/ \nsum(kube_pod_container_resource_requests{resource=\"cpu\",pod=~\".*vmstorage.*\"})\n", "hide": false, "legendFormat": "avg {{job}} usage", "range": true, "refId": "A" } ], "title": "CPU Usage (%)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of bytes read/write from the storage layer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read .*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 25 }, "id": 230, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{job}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{job}}", "range": true, "refId": "B" } ], "title": "Disk writes/reads ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of active time series with new data points inserted during the last hour across all storage nodes. High value may result in ingestion slowdown. \n\nSee following link for details:", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 33 }, "id": 12, "links": [ { "targetBlank": true, "title": "troubleshooting", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/blob/master/README.md#troubleshooting" } ], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(vm_cache_entries{job=~\"$job\", instance=~\"$instance\", type=\"storage/hour_metric_ids\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "Active time series", "refId": "A" } ], "title": "Active time series ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "How many datapoints are inserted into cluster per second by protocol before the replication.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 33 }, "id": 210, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "right", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "topk(10,sum(rate(vm_tenant_inserted_rows_total{job=~\"(vminsert-vmcluster|vmselect-vmcluster|vmstorage-vmcluster)\", instance=~\".*\"}[$__rate_interval])) by (type,accountID)) > 0", "interval": "", "legendFormat": "{{accountID}}/{{type}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_rows_inserted_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) > 0 ", "hide": false, "legendFormat": "total", "range": true, "refId": "B" } ], "title": "Ingestion rate by accountID(Top 10)", "type": "timeseries" } ], "title": "VM Stats", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$ds" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 29 }, "id": 10, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the rate and total number of new series created over last 24h.\n\nHigh churn rate tightly connected with database performance and may result in unexpected OOM's or slow queries. It is recommended to always keep an eye on this metric to avoid unexpected cardinality \"explosions\".\n\nThe higher churn rate is, the more resources required to handle it. Consider to keep the churn rate as low as possible.\n\nTo investigate stats about most expensive series use `api/v1/status/tsdb` handler. More details here https://docs.victoriametrics.com/Cluster-VictoriaMetrics.html#url-format\n\nGood references to read:\n* https://www.robustperception.io/cardinality-is-key\n* https://valyala.medium.com/high-cardinality-tsdb-benchmarks-victoriametrics-vs-timescaledb-vs-influxdb-13e6ee64dd6b", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "new series over 24h" }, "properties": [ { "id": "custom.axisPlacement", "value": "right" }, { "id": "custom.axisSoftMin", "value": 0 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "id": 102, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(rate(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval]))", "interval": "", "legendFormat": "churn rate", "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "exemplar": true, "expr": "sum(increase(vm_new_timeseries_created_total{job=~\"$job_storage\", instance=~\"$instance\"}[24h]))", "hide": false, "interval": "", "legendFormat": "new series over 24h", "refId": "B" } ], "title": "Churn rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the saturation level of connection between vminsert and vmstorage components. If the threshold of 0.9sec is reached, then the connection is saturated by more than 90% and vminsert won't be able to keep up. This usually means that more vminsert or vmstorage nodes must be added to the cluster in order to increase the total number of vminsert -> vmstorage links.\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0.9 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "id": 139, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(rate(vm_rpc_send_duration_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))by (addr)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Storage connection saturation ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The percentage of slow inserts comparing to total insertion rate during the last 5 minutes. \n\nThe less value is better. If percentage remains high (>10%) during extended periods of time, then it is likely more RAM is needed for optimal handling of the current number of active time series. \n\nIn general, VictoriaMetrics requires ~1KB or RAM per active time series, so it should be easy calculating the required amounts of RAM for the current workload according to capacity planning docs. But the resulting number may be far from the real number because the required amounts of memory depends on may other factors such as the number of labels per time series and the length of label values.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line+area" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "transparent" }, { "color": "red", "value": 0.1 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "id": 108, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n rate(vm_slow_row_inserts_total{job=~\"$job_storage\"}[$__rate_interval]) \n / rate(vm_rows_added_to_storage_total{job=~\"$job_storage\"}[$__rate_interval])\n)", "interval": "", "legendFormat": "slow inserts", "range": true, "refId": "A" } ], "title": "Slow inserts", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Slow queries according to `search.logSlowQueryDuration` flag, which is `5s` by default.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "id": 107, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "expr": "sum(rate(vm_slow_queries_total{job=~\"$job_select\", instance=~\"$instance\"}[$__rate_interval]))", "interval": "", "legendFormat": "slow queries rate", "refId": "A" } ], "title": "Slow queries rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 28 }, "id": 226, "options": { "legend": { "calcs": [ "lastNotNull", "mean" ], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "rate(vminsert_request_duration_seconds_sum[$__rate_interval]) / rate(vminsert_request_duration_seconds_count[$__rate_interval])", "legendFormat": "{{pod}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "expr": "avg(rate(vminsert_request_duration_seconds_sum[$__rate_interval]) / rate(vminsert_request_duration_seconds_count[$__rate_interval]))", "hide": false, "legendFormat": "overall", "range": true, "refId": "B" } ], "title": "Insert Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "The less time it takes is better.\n* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 28 }, "id": 8, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(vm_request_duration_seconds{job=~\"$job\", instance=~\"$instance\", quantile=\"0.99\"}) by (path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{path}}", "range": true, "refId": "A" } ], "title": "Query duration 0.99 quantile ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "* `*` - unsupported query path\n* `/write` - insert into VM\n* `/metrics` - query VM system metrics\n* `/query` - query instant values\n* `/query_range` - query over a range of time\n* `/series` - match a certain label set\n* `/label/{}/values` - query a list of label values (variables mostly)", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 36 }, "id": 6, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(vm_http_requests_total{job=~\"$job\", instance=~\"$instance\", path!~\"/favicon.ico\"}[$__rate_interval])) by (path) > 0", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{path}}", "range": true, "refId": "A" } ], "title": "Requests rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows cache miss ratio. Lower is better.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 12, "y": 36 }, "id": 58, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n rate(vm_cache_misses_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n rate(vm_cache_requests_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n) by(type)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache miss ratio ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used cache size from the allowed size by type. \nValues close to 100% show the maximum potential utilization.\nValues close to 0% show that cache is underutilized.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 9, "w": 12, "x": 0, "y": 44 }, "id": 144, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n vm_cache_size_bytes{job=~\"$job_storage\", instance=~\"$instance\"} \n /\n vm_cache_size_max_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(type)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache usage % by type ($instance)", "type": "timeseries" } ], "title": "Performance Metrics", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$ds" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, "id": 46, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 }, "id": 66, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RSS memory % usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Share for memory allocated by the process itself. When memory usage reaches 100% it will be likely OOM-killed.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=190&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 }, "id": 138, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "RSS anonymous memory % usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=${__field.labels.job}&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, "id": 64, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_cpu_cores{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "CPU ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the number of bytes read/write from the storage layer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "Bps" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/read .*/" }, "properties": [ { "id": "custom.transform", "value": "negative-Y" } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, "id": 122, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(process_io_storage_read_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{job}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(rate(process_io_storage_written_bytes_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job) > 0", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{job}}", "range": true, "refId": "B" } ], "title": "Disk writes/reads ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of open file descriptors compared to the limit set in the OS.\nReaching the limit of open files can cause various issues and must be prevented.\n\nSee how to change limits here https://medium.com/@muhammadtriwibowo/set-permanently-ulimit-n-open-files-in-ubuntu-4d61064429a", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 2, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/max.*/" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "id": 117, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_open_fds{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_max_fds{job=~\"$job\", instance=~\"$instance\"}\n) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{job}}", "range": true, "refId": "A" } ], "title": "Open FDs usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "id": 119, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(vm_tcplistener_conns{job=~\"$job\", instance=~\"$instance\"}[$__interval])) by(job)", "interval": "", "legendFormat": "{{job}}", "range": true, "refId": "A" } ], "title": "TCP connections ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "id": 68, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(go_goroutines{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Goroutines ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "id": 120, "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(rate(vm_tcplistener_accepts_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "TCP connections rate ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 0, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 37 }, "id": 70, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(max_over_time(process_num_threads{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Threads ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Percentage" }, "properties": [ { "id": "unit", "value": "percentunit" } ] }, { "matcher": { "id": "byName", "options": "persistentvolumeclaim" }, "properties": [ { "id": "custom.width", "value": 281 } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 37 }, "id": 239, "options": { "footer": { "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "Percentage" } ] }, "pluginVersion": "9.0.1", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "kubelet_volume_stats_capacity_bytes{namespace=\"observability\",persistentvolumeclaim=~\"vmstorage.*\"} - kubelet_volume_stats_available_bytes{namespace=\"observability\",persistentvolumeclaim=~\"vmstorage.*\"}", "format": "table", "instant": true, "legendFormat": "{{persistentvolumeclaim}}", "range": false, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS-REGIONAL-AP-NORTHEAST-1}" }, "editorMode": "code", "exemplar": false, "expr": "1 \n-\nkubelet_volume_stats_available_bytes{namespace=\"observability\",persistentvolumeclaim=~\"vmstorage.*\"}\n/\nkubelet_volume_stats_capacity_bytes{namespace=\"observability\",persistentvolumeclaim=~\"vmstorage.*\"} ", "format": "table", "hide": false, "instant": true, "legendFormat": "{{persistentvolumeclaim}}", "range": false, "refId": "B" } ], "title": "volume usage", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "endpoint": true, "instance": true, "job": true, "metrics_path": true, "namespace": true, "node": true, "persistentvolumeclaim": false, "service": true }, "indexByName": {}, "renameByName": { "Value #A": "Usage", "Value #B": "Percentage" } } } ], "type": "table" } ], "title": "Resource usage ($job)", "type": "row" }, { "collapsed": true, "datasource": { "type": "prometheus", "uid": "$ds" }, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 31 }, "id": 24, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=192&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 79 }, "id": 151, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "max(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "min(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "avg(\n rate(process_cpu_seconds_total{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n process_cpu_cores_available{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "range": true, "refId": "C" } ], "title": "CPU usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Percentage of used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=189&var-job=$job_storage&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 79 }, "id": 167, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "min(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "avg(\n max_over_time(process_resident_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}[$__rate_interval])\n /\n vm_available_memory_bytes{job=~\"$job_storage\", instance=~\"$instance\"}\n)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "range": true, "refId": "C" } ], "title": "Memory usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=200&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 86 }, "id": 20, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "max", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "min(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "min", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "avg(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) /\n (\n sum(vm_free_disk_space_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance) +\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n ) \n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "avg", "range": true, "refId": "C" } ], "title": "Disk space usage % ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the approx time needed to reach 100% of disk capacity for at least one vmstorage node based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=196&${__url_time_range}&${__all_variables}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 86 }, "id": 113, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "min(vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} \n/ \nignoring(path) (\n (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n - \n ignoring(type) rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\", type=\"merge\"}[1d])\n ) * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "min ETA", "range": true, "refId": "A" } ], "title": "Storage full ETA ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used disk space by type: datapoints or indexdb. Normally, indexdb takes much less space comparing to datapoints. But with high churn rate the size of the indexdb could grow significantly.\n\nThe sum of the % can be > 100% since panel shows max % per-job and per-instance. It means different instance can have different ratio between datapoints and indexdb size.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [ { "targetBlank": true, "title": "Drilldown", "url": "/d/oS7Bi_0Wz?viewPanel=201&var-ds=$ds&var-instance=$instance&${__url_time_range}" } ], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 94 }, "id": 202, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "indexdb", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "max(\n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)\n / \n sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\"}) by(job, instance)\n)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "datapoints", "range": true, "refId": "B" } ], "title": "Disk space usage % by type ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows how many ongoing insertions (not API /write calls) on disk are taking place, where:\n* `max` - equal to number of CPUs;\n* `current` - current number of goroutines busy with inserting rows into underlying storage.\n\nEvery successful API /write call results into flush on disk. The `max` is an internal limit and can't be changed. It is always equal to the number of CPUs. \n\nWhen `current` hits `max` constantly, it means storage is overloaded and requires more CPU or faster disk.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "max" }, "properties": [ { "id": "color", "value": { "fixedColor": "#C4162A", "mode": "fixed" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 94 }, "id": 133, "links": [ { "targetBlank": true, "title": "Related discussion", "url": "https://github.com/VictoriaMetrics/VictoriaMetrics/issues/632" } ], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "sum(avg_over_time(vm_concurrent_addrows_current{job=~\"$job_storage\", instance=~\"$instance\"}[1m])) by(instance)", "interval": "", "legendFormat": "current", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max(vm_concurrent_addrows_capacity{job=~\"$job_storage\", instance=~\"$instance\"})", "hide": false, "interval": "", "legendFormat": "max", "range": true, "refId": "B" } ], "title": "Concurrent flushes on disk ($instance)", "type": "timeseries" } ], "title": "vmstorage ($instance)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 32 }, "id": 194, "panels": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "gridPos": { "h": 2, "w": 24, "x": 0, "y": 90 }, "id": 198, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "Drilldown row is used by other panels on the dashboard to show more detailed metrics per-instance.", "mode": "markdown" }, "pluginVersion": "9.0.1", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the used memory (resident).\nThe application's performance will significantly degrade when memory usage is close to 100%.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 92 }, "id": 189, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max_over_time(process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "RSS memory usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "RSS share for memory allocated by the process itself. This share cannot be freed by the OS, so it must be taken into account by OOM killer.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 92 }, "id": 190, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": true, "expr": "max_over_time(process_resident_memory_anon_bytes{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])", "interval": "", "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "RSS anonymous memory usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 100 }, "id": 192, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])) by(job, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} ({{job}})", "range": true, "refId": "A" } ], "title": "CPU usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the approx time needed to reach 100% of disk capacity based on the following params:\n* free disk space;\n* row ingestion rate;\n* dedup rate;\n* compression.\n\nUse this panel for capacity planning in order to estimate the time remaining for running out of the disk space.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 100 }, "id": 196, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "min" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "vm_free_disk_space_bytes{job=~\"$job_storage\", instance=~\"$instance\"} \n/ \nignoring(path) (\n (\n rate(vm_rows_added_to_storage_total{job=~\"$job_storage\", instance=~\"$instance\"}[1d])\n - \n ignoring(type) rate(vm_deduplicated_samples_total{job=~\"$job_storage\", instance=~\"$instance\", type=\"merge\"}[1d])\n ) * scalar(\n sum(vm_data_size_bytes{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n / \n sum(vm_rows{job=~\"$job_storage\", instance=~\"$instance\", type!~\"indexdb.*\"})\n )\n)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Storage full ETA ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "Shows the percentage of used disk space. It is recommended to have at least 20% of free disk space for the best performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 107 }, "id": 200, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": false }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "1\n-\n(\n sum(kubelet_volume_stats_available_bytes{persistentvolumeclaim=~\"vmstorage.*\"})by (persistentvolumeclaim)\n /\n sum(kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~\"vmstorage.*\"}) by (persistentvolumeclaim)\n)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{persistentvolumeclaim}}", "range": true, "refId": "A" } ], "title": "Disk space usage ($instance)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "line" } }, "links": [], "mappings": [], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 107 }, "id": 201, "links": [], "options": { "legend": { "calcs": [ "mean", "lastNotNull", "max" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "9.1.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type=~\"indexdb.*\"}) by(job, instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{job}}:{{instance}} (indexdb)", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "$ds" }, "editorMode": "code", "expr": "sum(vm_data_size_bytes{job=~\"$job\", instance=~\"$instance\", type!~\"indexdb.*\"}) by(job, instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{job}}:{{instance}} (datapoints)", "range": true, "refId": "B" } ], "title": "Disk space usage by type ($instance)", "type": "timeseries" } ], "title": "Drilldown", "type": "row" } ], "refresh": false, "schemaVersion": 36, "style": "dark", "tags": [], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus-regional-ap-northeast-1", "value": "Prometheus-regional-ap-northeast-1" }, "hide": 0, "includeAll": false, "multi": false, "name": "ds", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "^(?!.*(seed|serverless)$).*", "skipUrlSync": false, "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", "hide": 0, "includeAll": true, "multi": true, "name": "job", "options": [], "query": { "query": "label_values(vm_app_version{version=~\"^vm(insert|select|storage).*\"}, job)", "refId": "VictoriaMetrics-job-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", "hide": 2, "includeAll": false, "multi": false, "name": "job_insert", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vminsert.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", "hide": 2, "includeAll": false, "multi": false, "name": "job_select", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmselect.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "current": {}, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", "hide": 2, "includeAll": false, "multi": false, "name": "job_storage", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\", version=~\"^vmstorage.*\"}, job)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": {}, "datasource": { "type": "prometheus", "uid": "$ds" }, "definition": "label_values(vm_app_version{job=~\"$job\"}, instance)", "hide": 0, "includeAll": true, "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(vm_app_version{job=~\"$job\"}, instance)", "refId": "VictoriaMetrics-instance-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "current": { "selected": false, "text": "Prometheus-regional-ap-northeast-1-seed", "value": "Prometheus-regional-ap-northeast-1-seed" }, "hide": 2, "includeAll": false, "multi": false, "name": "ds_seed", "options": [], "query": "prometheus", "queryValue": "", "refresh": 1, "regex": "$ds-seed", "skipUrlSync": false, "type": "datasource" }, { "current": { "selected": false, "text": "Prometheus-regional-ap-northeast-1-serverless", "value": "Prometheus-regional-ap-northeast-1-serverless" }, "hide": 2, "includeAll": false, "multi": false, "name": "ds_serverless", "options": [], "query": "prometheus", "refresh": 1, "regex": "$ds-serverless", "skipUrlSync": false, "type": "datasource" } ] }, "time": { "from": "now-5m", "to": "now" }, "timepicker": { "refresh_intervals": [ "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ] }, "timezone": "", "title": "VictoriaMetrics Capacity Planning", "uid": "0Wz_just1900", "version": 8, "weekStart": "" }