{ "duration_msec": 1201.014, "message": "/select/4030/prometheus/api/v1/query: query=(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~\"tmpfs\"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0, time=1688543731773: series=0", "children": [ { "duration_msec": 1200.901, "message": "eval: query=((((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes) < 10) and on (instance, device, mountpoint) (predict_linear(node_filesystem_avail_bytes{fstype!~\"tmpfs\"}[1h], 86400) < 0)) and on (instance, device, mountpoint) (node_filesystem_readonly == 0), timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=0, points=0, pointsPerSeries=0", "children": [ { "duration_msec": 1200.895, "message": "binary op \"and\": series=0", "children": [ { "duration_msec": 291.248, "message": "eval: query=node_filesystem_readonly == 0, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=800, points=800, pointsPerSeries=1", "children": [ { "duration_msec": 291.245, "message": "binary op \"==\": series=800", "children": [ { "duration_msec": 284.244, "message": "eval: query=node_filesystem_readonly, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=800, points=800, pointsPerSeries=1", "children": [ { "duration_msec": 284.233, "message": "rollup default_rollup(): timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0: neededMemoryBytes=12800", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=node_filesystem_readonly, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0", "children": [ { "duration_msec": 0, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 206.491, "message": "fetch matching series: accountID=4030, projectID=0, filters=[{__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 198.111, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 150.377, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.677, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.434, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.07, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.065, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.062, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 406 bytes" }, { "duration_msec": 0.019, "message": "decompressed tsids to 6952 bytes" }, { "duration_msec": 0.03, "message": "unmarshaled 217 tsids" } ] } ] } ] }, { "duration_msec": 0.029, "message": "prefetch metric names for 217 tsids", "children": [ { "duration_msec": 0.026, "message": "217 out of 217 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=217" } ] }, { "duration_msec": 0.321, "message": "search for parts with data for 217 series" } ] }, { "duration_msec": 3.146, "message": "sent 840 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 175.407, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 127.604, "message": "rpc call search_v7()", "children": [ { "duration_msec": 5.335, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.406, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.093, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.089, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.086, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.007, "message": "found tsids with compressed size: 410 bytes" }, { "duration_msec": 0.017, "message": "decompressed tsids to 6952 bytes" }, { "duration_msec": 0.059, "message": "unmarshaled 217 tsids" } ] } ] } ] }, { "duration_msec": 0.029, "message": "prefetch metric names for 217 tsids", "children": [ { "duration_msec": 0.026, "message": "217 out of 217 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=217" } ] }, { "duration_msec": 1.265, "message": "search for parts with data for 217 series" } ] }, { "duration_msec": 3.856, "message": "sent 2152 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 151.674, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 103.839, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.169, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.051, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.063, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.058, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.055, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 368 bytes" }, { "duration_msec": 0.034, "message": "decompressed tsids to 5992 bytes" }, { "duration_msec": 0.009, "message": "unmarshaled 187 tsids" } ] } ] } ] }, { "duration_msec": 0.027, "message": "prefetch metric names for 187 tsids", "children": [ { "duration_msec": 0.024, "message": "187 out of 187 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=187" } ] }, { "duration_msec": 0.925, "message": "search for parts with data for 187 series" } ] }, { "duration_msec": 2.012, "message": "sent 704 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 153.838, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 105.954, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.22, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.803, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.063, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.059, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.057, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.026, "message": "found tsids with compressed size: 387 bytes" }, { "duration_msec": 0.019, "message": "decompressed tsids to 6600 bytes" }, { "duration_msec": 0.009, "message": "unmarshaled 206 tsids" } ] } ] } ] }, { "duration_msec": 0.025, "message": "prefetch metric names for 206 tsids", "children": [ { "duration_msec": 0.022, "message": "206 out of 206 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=206" } ] }, { "duration_msec": 0.698, "message": "search for parts with data for 206 series" } ] }, { "duration_msec": 2.351, "message": "sent 893 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 199.49, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 151.56, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.79, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.133, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.054, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.049, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.046, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 417 bytes" }, { "duration_msec": 0.019, "message": "decompressed tsids to 7272 bytes" }, { "duration_msec": 0.014, "message": "unmarshaled 227 tsids" } ] } ] } ] }, { "duration_msec": 0.036, "message": "prefetch metric names for 227 tsids", "children": [ { "duration_msec": 0.026, "message": "227 out of 227 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=227" } ] }, { "duration_msec": 1.013, "message": "search for parts with data for 227 series" } ] }, { "duration_msec": 2.541, "message": "sent 1338 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 129.863, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 81.884, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.488, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.929, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.045, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.04, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.037, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 389 bytes" }, { "duration_msec": 0.016, "message": "decompressed tsids to 6824 bytes" }, { "duration_msec": 0.009, "message": "unmarshaled 213 tsids" } ] } ] } ] }, { "duration_msec": 0.041, "message": "prefetch metric names for 213 tsids", "children": [ { "duration_msec": 0.037, "message": "213 out of 213 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=213" } ] }, { "duration_msec": 0.828, "message": "search for parts with data for 213 series" } ] }, { "duration_msec": 1.495, "message": "sent 588 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 203.373, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 155.367, "message": "rpc call search_v7()", "children": [ { "duration_msec": 6.592, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.724, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.071, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.067, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.064, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 383 bytes" }, { "duration_msec": 0.034, "message": "decompressed tsids to 6696 bytes" }, { "duration_msec": 0.018, "message": "unmarshaled 209 tsids" } ] } ] } ] }, { "duration_msec": 0.028, "message": "prefetch metric names for 209 tsids", "children": [ { "duration_msec": 0.026, "message": "209 out of 209 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=209" } ] }, { "duration_msec": 1.55, "message": "search for parts with data for 209 series" } ] }, { "duration_msec": 4.697, "message": "sent 2270 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 87.567, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 87.559, "message": "rpc call search_v7()", "children": [ { "duration_msec": 15.251, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.471, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.041, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.037, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.035, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 389 bytes" }, { "duration_msec": 0.016, "message": "decompressed tsids to 6536 bytes" }, { "duration_msec": 0.008, "message": "unmarshaled 204 tsids" } ] } ] } ] }, { "duration_msec": 0.029, "message": "prefetch metric names for 204 tsids", "children": [ { "duration_msec": 0.026, "message": "204 out of 204 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=204" } ] }, { "duration_msec": 1.381, "message": "search for parts with data for 204 series" } ] }, { "duration_msec": 13.714, "message": "sent 1911 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 118.481, "message": "fetch unique series=800, blocks=10696, samples=150501, bytes=1089915" } ] }, { "duration_msec": 77.632, "message": "rollup default_rollup() over 800 series; rollupConfigs=[timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0, points=1]", "children": [ { "duration_msec": 77.617, "message": "parallel process of fetched data: series=800, samples=32000" }, { "duration_msec": 0.003, "message": "samplesScanned=32800" } ] }, { "duration_msec": 0.002, "message": "rollup cache put: query=node_filesystem_readonly, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0, series=800", "children": [ { "duration_msec": 0, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] }, { "duration_msec": 0.011, "message": "eval: query=0, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] }, { "duration_msec": 899.165, "message": "eval: query=(((node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host-10|host-11|host-12|host-13|host-14|ho../shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400) < 0), timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=0, points=0, pointsPerSeries=0", "children": [ { "duration_msec": 899.158, "message": "binary op \"and\": series=0", "children": [ { "duration_msec": 350.092, "message": "eval: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..6/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400) < 0, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 350.086, "message": "binary op \"<\": series=300", "children": [ { "duration_msec": 348.028, "message": "eval: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..bf186/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400), timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 0.004, "message": "eval: query=86400, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" }, { "duration_msec": 348.005, "message": "rollup predict_linear(): timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=3600000: neededMemoryBytes=4800", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..bf186/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400), timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=3600000", "children": [ { "duration_msec": 0.001, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 239.246, "message": "fetch matching series: accountID=4030, projectID=0, filters=[{__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!~\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 142.766, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 140.543, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.95, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.346, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.039, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.025, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.02, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.007, "message": "found tsids with compressed size: 240 bytes" }, { "duration_msec": 0.007, "message": "decompressed tsids to 2152 bytes" }, { "duration_msec": 0.003, "message": "unmarshaled 67 tsids" } ] } ] } ] }, { "duration_msec": 0.015, "message": "prefetch metric names for 67 tsids", "children": [ { "duration_msec": 0.013, "message": "67 out of 67 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=67" } ] }, { "duration_msec": 0.271, "message": "search for parts with data for 67 series" } ] }, { "duration_msec": 1.506, "message": "sent 469 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 191.896, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 189.645, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.98, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.036, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.041, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.026, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.02, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 251 bytes" }, { "duration_msec": 0.007, "message": "decompressed tsids to 2216 bytes" }, { "duration_msec": 0.004, "message": "unmarshaled 69 tsids" } ] } ] } ] }, { "duration_msec": 0.018, "message": "prefetch metric names for 69 tsids", "children": [ { "duration_msec": 0.016, "message": "69 out of 69 metric names must be pre-fetched" }, { "duration_msec": 0, "message": "skip pre-fetching metric names for low number of metrid ids=69" } ] }, { "duration_msec": 0.957, "message": "search for parts with data for 69 series" } ] }, { "duration_msec": 2.841, "message": "sent 1110 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 176.362, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 174.085, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.785, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.192, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.077, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.06, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.056, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.017, "message": "found tsids with compressed size: 208 bytes" }, { "duration_msec": 0.03, "message": "decompressed tsids to 2504 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 78 tsids" } ] } ] } ] }, { "duration_msec": 0.018, "message": "prefetch metric names for 78 tsids", "children": [ { "duration_msec": 0.016, "message": "78 out of 78 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=78" } ] }, { "duration_msec": 1.074, "message": "search for parts with data for 78 series" } ] }, { "duration_msec": 1.439, "message": "sent 469 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 172.821, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 170.518, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.825, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.769, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.051, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.034, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.03, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 223 bytes" }, { "duration_msec": 0.013, "message": "decompressed tsids to 3080 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 96 tsids" } ] } ] } ] }, { "duration_msec": 0.015, "message": "prefetch metric names for 96 tsids", "children": [ { "duration_msec": 0.013, "message": "96 out of 96 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=96" } ] }, { "duration_msec": 0.671, "message": "search for parts with data for 96 series" } ] }, { "duration_msec": 1.946, "message": "sent 726 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 180.063, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 177.724, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.814, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.206, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.045, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.028, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.026, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 209 bytes" }, { "duration_msec": 0.012, "message": "decompressed tsids to 2696 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 84 tsids" } ] } ] } ] }, { "duration_msec": 0.016, "message": "prefetch metric names for 84 tsids", "children": [ { "duration_msec": 0.014, "message": "84 out of 84 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=84" } ] }, { "duration_msec": 1.118, "message": "search for parts with data for 84 series" } ] }, { "duration_msec": 1.502, "message": "sent 596 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 131.58, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 129.219, "message": "rpc call search_v7()", "children": [ { "duration_msec": 9.171, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.705, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.045, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.025, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.022, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 240 bytes" }, { "duration_msec": 0.009, "message": "decompressed tsids to 2120 bytes" }, { "duration_msec": 0.003, "message": "unmarshaled 66 tsids" } ] } ] } ] }, { "duration_msec": 0.014, "message": "prefetch metric names for 66 tsids", "children": [ { "duration_msec": 0.012, "message": "66 out of 66 metric names must be pre-fetched" }, { "duration_msec": 0, "message": "skip pre-fetching metric names for low number of metrid ids=66" } ] }, { "duration_msec": 0.626, "message": "search for parts with data for 66 series" } ] }, { "duration_msec": 8.358, "message": "sent 321 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 174.732, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 172.348, "message": "rpc call search_v7()", "children": [ { "duration_msec": 57.549, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.608, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.039, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.024, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.02, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 252 bytes" }, { "duration_msec": 0.007, "message": "decompressed tsids to 2280 bytes" }, { "duration_msec": 0.004, "message": "unmarshaled 71 tsids" } ] } ] } ] }, { "duration_msec": 0.039, "message": "prefetch metric names for 71 tsids", "children": [ { "duration_msec": 0.036, "message": "71 out of 71 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=71" } ] }, { "duration_msec": 1.504, "message": "search for parts with data for 71 series" } ] }, { "duration_msec": 55.782, "message": "sent 964 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 137.051, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 137.043, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.497, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.995, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.053, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:50:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.035, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.029, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.007, "message": "found tsids with compressed size: 247 bytes" }, { "duration_msec": 0.015, "message": "decompressed tsids to 2216 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 69 tsids" } ] } ] } ] }, { "duration_msec": 0.021, "message": "prefetch metric names for 69 tsids", "children": [ { "duration_msec": 0.019, "message": "69 out of 69 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=69" } ] }, { "duration_msec": 0.897, "message": "search for parts with data for 69 series" } ] }, { "duration_msec": 2.36, "message": "sent 812 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 59.802, "message": "fetch unique series=300, blocks=5467, samples=264981, bytes=946014" } ] }, { "duration_msec": 108.474, "message": "rollup predict_linear() over 300 series; rollupConfigs=[timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=3600000, points=1]", "children": [ { "duration_msec": 108.457, "message": "parallel process of fetched data: series=300, samples=77910" }, { "duration_msec": 0.003, "message": "samplesScanned=148320" } ] }, { "duration_msec": 0.002, "message": "rollup cache put: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..bf186/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400), timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=3600000, series=300", "children": [ { "duration_msec": 0, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] }, { "duration_msec": 0.004, "message": "eval: query=0, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] }, { "duration_msec": 465.049, "message": "eval: query=((node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|ho../flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}) < 10, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 465.044, "message": "binary op \"<\": series=300", "children": [ { "duration_msec": 462.328, "message": "eval: query=(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|hos..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 462.32, "message": "binary op \"/\": series=300", "children": [ { "duration_msec": 253.903, "message": "eval: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host../flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"} * 100, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 253.9, "message": "binary op \"*\": series=300", "children": [ { "duration_msec": 251.85, "message": "eval: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 251.84, "message": "rollup default_rollup(): timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0: neededMemoryBytes=4800", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0", "children": [ { "duration_msec": 0, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 234.407, "message": "fetch matching series: accountID=4030, projectID=0, filters=[{__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 135.782, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 97.168, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.3, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.372, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.066, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.04, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.032, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 240 bytes" }, { "duration_msec": 0.013, "message": "decompressed tsids to 2152 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 67 tsids" } ] } ] } ] }, { "duration_msec": 0.015, "message": "prefetch metric names for 67 tsids", "children": [ { "duration_msec": 0.012, "message": "67 out of 67 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=67" } ] }, { "duration_msec": 0.263, "message": "search for parts with data for 67 series" } ] }, { "duration_msec": 0.741, "message": "sent 268 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 133.781, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 95.089, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.084, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.136, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.062, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.036, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.031, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.01, "message": "found tsids with compressed size: 251 bytes" }, { "duration_msec": 0.012, "message": "decompressed tsids to 2216 bytes" }, { "duration_msec": 0.004, "message": "unmarshaled 69 tsids" } ] } ] } ] }, { "duration_msec": 0.016, "message": "prefetch metric names for 69 tsids", "children": [ { "duration_msec": 0.013, "message": "69 out of 69 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=69" } ] }, { "duration_msec": 0.997, "message": "search for parts with data for 69 series" } ] }, { "duration_msec": 1.746, "message": "sent 696 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 131.835, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 93.105, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.009, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.078, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.067, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.039, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.03, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.01, "message": "found tsids with compressed size: 208 bytes" }, { "duration_msec": 0.013, "message": "decompressed tsids to 2504 bytes" }, { "duration_msec": 0.004, "message": "unmarshaled 78 tsids" } ] } ] } ] }, { "duration_msec": 0.016, "message": "prefetch metric names for 78 tsids", "children": [ { "duration_msec": 0.013, "message": "78 out of 78 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=78" } ] }, { "duration_msec": 0.959, "message": "search for parts with data for 78 series" } ] }, { "duration_msec": 0.751, "message": "sent 313 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 135.115, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 96.357, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.81, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.752, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.065, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.04, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.033, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 223 bytes" }, { "duration_msec": 0.015, "message": "decompressed tsids to 3080 bytes" }, { "duration_msec": 0.006, "message": "unmarshaled 96 tsids" } ] } ] } ] }, { "duration_msec": 0.017, "message": "prefetch metric names for 96 tsids", "children": [ { "duration_msec": 0.014, "message": "96 out of 96 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=96" } ] }, { "duration_msec": 0.641, "message": "search for parts with data for 96 series" } ] }, { "duration_msec": 0.924, "message": "sent 438 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 233.615, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 194.812, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.588, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.076, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.053, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.03, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.026, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 209 bytes" }, { "duration_msec": 0.012, "message": "decompressed tsids to 2696 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 84 tsids" } ] } ] } ] }, { "duration_msec": 0.015, "message": "prefetch metric names for 84 tsids", "children": [ { "duration_msec": 0.013, "message": "84 out of 84 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=84" } ] }, { "duration_msec": 0.976, "message": "search for parts with data for 84 series" } ] }, { "duration_msec": 1.371, "message": "sent 512 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 130.632, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 91.804, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.527, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.951, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.058, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.036, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.033, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.016, "message": "found tsids with compressed size: 240 bytes" }, { "duration_msec": 0.009, "message": "decompressed tsids to 2120 bytes" }, { "duration_msec": 0.003, "message": "unmarshaled 66 tsids" } ] } ] } ] }, { "duration_msec": 0.021, "message": "prefetch metric names for 66 tsids", "children": [ { "duration_msec": 0.019, "message": "66 out of 66 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=66" } ] }, { "duration_msec": 0.844, "message": "search for parts with data for 66 series" } ] }, { "duration_msec": 0.444, "message": "sent 189 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 178.655, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 139.786, "message": "rpc call search_v7()", "children": [ { "duration_msec": 13.984, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.084, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.071, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.048, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.044, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 252 bytes" }, { "duration_msec": 0.029, "message": "decompressed tsids to 2280 bytes" }, { "duration_msec": 0.004, "message": "unmarshaled 71 tsids" } ] } ] } ] }, { "duration_msec": 0.02, "message": "prefetch metric names for 71 tsids", "children": [ { "duration_msec": 0.017, "message": "71 out of 71 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=71" } ] }, { "duration_msec": 0.956, "message": "search for parts with data for 71 series" } ] }, { "duration_msec": 12.716, "message": "sent 822 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 77.692, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 77.684, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.544, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.633, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.061, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.034, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.027, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 247 bytes" }, { "duration_msec": 0.009, "message": "decompressed tsids to 2216 bytes" }, { "duration_msec": 0.006, "message": "unmarshaled 69 tsids" } ] } ] } ] }, { "duration_msec": 0.016, "message": "prefetch metric names for 69 tsids", "children": [ { "duration_msec": 0.013, "message": "69 out of 69 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=69" } ] }, { "duration_msec": 0.516, "message": "search for parts with data for 69 series" } ] }, { "duration_msec": 1.774, "message": "sent 674 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 156.577, "message": "fetch unique series=300, blocks=3912, samples=56645, bytes=484537" } ] }, { "duration_msec": 17.064, "message": "rollup default_rollup() over 300 series; rollupConfigs=[timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0, points=1]", "children": [ { "duration_msec": 17.045, "message": "parallel process of fetched data: series=300, samples=12000" }, { "duration_msec": 0.004, "message": "samplesScanned=12300" } ] }, { "duration_msec": 0.002, "message": "rollup cache put: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0, series=300", "children": [ { "duration_msec": 0, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] }, { "duration_msec": 0.004, "message": "eval: query=100, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] }, { "duration_msec": 202.145, "message": "eval: query=node_filesystem_size_bytes{cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\", cluster_id=\"1059048344677\", component=\"node_exporter\", device=~\"/dev/root|..der=\"gcp\", seed_region=\"us-west1\", shoot_infra=\"gke\", shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\", tenant=\"benchmark\", tidb_cluster_id=\"large\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=300, points=300, pointsPerSeries=1", "children": [ { "duration_msec": 202.134, "message": "rollup default_rollup(): timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0: neededMemoryBytes=4800", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=node_filesystem_size_bytes{cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\", cluster_id=\"1059048344677\", component=\"node_exporter\", device=~\"/dev/root|..der=\"gcp\", seed_region=\"us-west1\", shoot_infra=\"gke\", shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\", tenant=\"benchmark\", tidb_cluster_id=\"large\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0", "children": [ { "duration_msec": 0, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 164.523, "message": "fetch matching series: accountID=4030, projectID=0, filters=[{__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 47.854, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 47.748, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.347, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.42, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.075, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.039, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.032, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 253 bytes" }, { "duration_msec": 0.016, "message": "decompressed tsids to 2280 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 71 tsids" } ] } ] } ] }, { "duration_msec": 0.017, "message": "prefetch metric names for 71 tsids", "children": [ { "duration_msec": 0.015, "message": "71 out of 71 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=71" } ] }, { "duration_msec": 0.267, "message": "search for parts with data for 71 series" } ] }, { "duration_msec": 0.698, "message": "sent 284 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 46.076, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 44.548, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.919, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.36, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.075, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.034, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.026, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 251 bytes" }, { "duration_msec": 0.01, "message": "decompressed tsids to 2248 bytes" }, { "duration_msec": 0.005, "message": "unmarshaled 70 tsids" } ] } ] } ] }, { "duration_msec": 0.017, "message": "prefetch metric names for 70 tsids", "children": [ { "duration_msec": 0.014, "message": "70 out of 70 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=70" } ] }, { "duration_msec": 1.214, "message": "search for parts with data for 70 series" } ] }, { "duration_msec": 1.317, "message": "sent 710 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 65.439, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 63.776, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.712, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.245, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.096, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.037, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.03, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 211 bytes" }, { "duration_msec": 0.012, "message": "decompressed tsids to 2632 bytes" }, { "duration_msec": 0.006, "message": "unmarshaled 82 tsids" } ] } ] } ] }, { "duration_msec": 0.063, "message": "prefetch metric names for 82 tsids", "children": [ { "duration_msec": 0.059, "message": "82 out of 82 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=82" } ] }, { "duration_msec": 0.986, "message": "search for parts with data for 82 series" } ] }, { "duration_msec": 1.153, "message": "sent 328 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 46.933, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 45.232, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.964, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 0.898, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.1, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.056, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.05, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.025, "message": "found tsids with compressed size: 207 bytes" }, { "duration_msec": 0.015, "message": "decompressed tsids to 2504 bytes" }, { "duration_msec": 0.007, "message": "unmarshaled 78 tsids" } ] } ] } ] }, { "duration_msec": 0.017, "message": "prefetch metric names for 78 tsids", "children": [ { "duration_msec": 0.015, "message": "78 out of 78 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=78" } ] }, { "duration_msec": 0.737, "message": "search for parts with data for 78 series" } ] }, { "duration_msec": 0.79, "message": "sent 351 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 163.793, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 162.047, "message": "rpc call search_v7()", "children": [ { "duration_msec": 2.39, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.205, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.077, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.033, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.027, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 249 bytes" }, { "duration_msec": 0.012, "message": "decompressed tsids to 2184 bytes" }, { "duration_msec": 0.004, "message": "unmarshaled 68 tsids" } ] } ] } ] }, { "duration_msec": 0.017, "message": "prefetch metric names for 68 tsids", "children": [ { "duration_msec": 0.015, "message": "68 out of 68 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=68" } ] }, { "duration_msec": 1.065, "message": "search for parts with data for 68 series" } ] }, { "duration_msec": 0.96, "message": "sent 416 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 45.99, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 44.215, "message": "rpc call search_v7()", "children": [ { "duration_msec": 1.504, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.008, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.096, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.045, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.037, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.01, "message": "found tsids with compressed size: 203 bytes" }, { "duration_msec": 0.014, "message": "decompressed tsids to 2312 bytes" }, { "duration_msec": 0.007, "message": "unmarshaled 72 tsids" } ] } ] } ] }, { "duration_msec": 0.021, "message": "prefetch metric names for 72 tsids", "children": [ { "duration_msec": 0.017, "message": "72 out of 72 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=72" } ] }, { "duration_msec": 0.842, "message": "search for parts with data for 72 series" } ] }, { "duration_msec": 0.296, "message": "sent 209 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 76.711, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 74.909, "message": "rpc call search_v7()", "children": [ { "duration_msec": 5.007, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 2.249, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.131, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.042, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.034, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 207 bytes" }, { "duration_msec": 0.011, "message": "decompressed tsids to 2536 bytes" }, { "duration_msec": 0.009, "message": "unmarshaled 79 tsids" } ] } ] } ] }, { "duration_msec": 0.067, "message": "prefetch metric names for 79 tsids", "children": [ { "duration_msec": 0.064, "message": "79 out of 79 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=79" } ] }, { "duration_msec": 2.001, "message": "search for parts with data for 79 series" } ] }, { "duration_msec": 2.537, "message": "sent 905 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 51.569, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 51.561, "message": "rpc call search_v7()", "children": [ { "duration_msec": 3.311, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 1.498, "message": "init series search: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.074, "message": "search for matching tsids: filters=[{AccountID=4030,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",cluster_id=\"1059048344677\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-11..5|host-6|host-7|host-8|host-9\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r11|r19|r23|r40|r43|r44\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:45:31.773Z..2023-07-05T07:55:31.773Z]", "children": [ { "duration_msec": 0.031, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.026, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 209 bytes" }, { "duration_msec": 0.011, "message": "decompressed tsids to 2568 bytes" }, { "duration_msec": 0.006, "message": "unmarshaled 80 tsids" } ] } ] } ] }, { "duration_msec": 0.021, "message": "prefetch metric names for 80 tsids", "children": [ { "duration_msec": 0.015, "message": "80 out of 80 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=80" } ] }, { "duration_msec": 1.353, "message": "search for parts with data for 80 series" } ] }, { "duration_msec": 1.623, "message": "sent 782 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 112.805, "message": "fetch unique series=300, blocks=3985, samples=55489, bytes=406242" } ] }, { "duration_msec": 37.325, "message": "rollup default_rollup() over 300 series; rollupConfigs=[timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0, points=1]", "children": [ { "duration_msec": 37.316, "message": "parallel process of fetched data: series=300, samples=12000" }, { "duration_msec": 0.002, "message": "samplesScanned=12300" } ] }, { "duration_msec": 0.006, "message": "rollup cache put: query=node_filesystem_size_bytes{cluster=\"stag-benchmark-gcp-us-west1-eb5e4eb0\", cluster_id=\"1059048344677\", component=\"node_exporter\", device=~\"/dev/root|..der=\"gcp\", seed_region=\"us-west1\", shoot_infra=\"gke\", shoot_name=\"stag-benchmark-gcp-us-west1-eb5e4eb0\", tenant=\"benchmark\", tidb_cluster_id=\"large\"}, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, window=0, series=300", "children": [ { "duration_msec": 0.001, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] } ] } ] }, { "duration_msec": 0.004, "message": "eval: query=10, timeRange=[2023-07-05T07:55:31.773Z..2023-07-05T07:55:31.773Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] } ] } ] } ] } ] }, { "duration_msec": 0.003, "message": "leave only the first point in every series" }, { "duration_msec": 0.004, "message": "sort series by metric name and labels" }, { "duration_msec": 0.022, "message": "generate /api/v1/query response for series=0" } ] }