{ "duration_msec": 13093.037, "message": "/select/3850/prometheus/api/v1/query: query=(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) predict_linear(node_filesystem_avail_bytes{fstype!~\"tmpfs\"}[1h], 24 * 3600) < 0 and ON (instance, device, mountpoint) node_filesystem_readonly == 0, time=1688541765230: series=0", "children": [ { "duration_msec": 290.47, "message": "wait in queue because -search.maxConcurrentRequests=16 concurrent requests are executed" }, { "duration_msec": 12802.494, "message": "eval: query=((((node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes) < 10) and on (instance, device, mountpoint) (predict_linear(node_filesystem_avail_bytes{fstype!~\"tmpfs\"}[1h], 86400) < 0)) and on (instance, device, mountpoint) (node_filesystem_readonly == 0), timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=0, points=0, pointsPerSeries=0", "children": [ { "duration_msec": 12802.489, "message": "binary op \"and\": series=0", "children": [ { "duration_msec": 6333.928, "message": "eval: query=node_filesystem_readonly == 0, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=18000, points=18000, pointsPerSeries=1", "children": [ { "duration_msec": 6333.924, "message": "binary op \"==\": series=18000", "children": [ { "duration_msec": 5195.586, "message": "eval: query=node_filesystem_readonly, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=18000, points=18000, pointsPerSeries=1", "children": [ { "duration_msec": 5195.578, "message": "rollup default_rollup(): timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0: neededMemoryBytes=288000", "children": [ { "duration_msec": 0.001, "message": "rollup cache get: query=node_filesystem_readonly, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0", "children": [ { "duration_msec": 0, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 1459.22, "message": "fetch matching series: accountID=3850, projectID=0, filters=[{__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 133.139, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 132.767, "message": "rpc call search_v7()", "children": [ { "duration_msec": 41.53, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.69, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.504, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.5, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.498, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.008, "message": "found tsids with compressed size: 6028 bytes" }, { "duration_msec": 0.314, "message": "decompressed tsids to 152584 bytes" }, { "duration_msec": 0.168, "message": "unmarshaled 4768 tsids" } ] } ] } ] }, { "duration_msec": 0.284, "message": "prefetch metric names for 4768 tsids", "children": [ { "duration_msec": 0.261, "message": "483 out of 4768 metric names must be pre-fetched" }, { "duration_msec": 0.02, "message": "skip pre-fetching metric names for low number of metrid ids=483" } ] }, { "duration_msec": 3.88, "message": "search for parts with data for 4768 series" } ] }, { "duration_msec": 36.739, "message": "sent 9121 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 919.093, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 918.682, "message": "rpc call search_v7()", "children": [ { "duration_msec": 228.918, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 10.696, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.652, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.647, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.644, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.012, "message": "found tsids with compressed size: 6030 bytes" }, { "duration_msec": 0.249, "message": "decompressed tsids to 151816 bytes" }, { "duration_msec": 0.378, "message": "unmarshaled 4744 tsids" } ] } ] } ] }, { "duration_msec": 0.248, "message": "prefetch metric names for 4744 tsids", "children": [ { "duration_msec": 0.244, "message": "467 out of 4744 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=467" } ] }, { "duration_msec": 9.772, "message": "search for parts with data for 4744 series" } ] }, { "duration_msec": 218.089, "message": "sent 29318 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 1162.854, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 1162.423, "message": "rpc call search_v7()", "children": [ { "duration_msec": 170.508, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 25.004, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 20.846, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 14.679, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.03, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.029, "message": "cache miss" } ] }, { "duration_msec": 0.527, "message": "search for metric ids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z], maxMetrics=300000", "children": [ { "duration_msec": 0.521, "message": "update metric ids: filters={AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}, timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]: updated 4555 metric ids", "children": [ { "duration_msec": 0.514, "message": "search for metric ids on a particular day: filters={AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}, date=2023-07-05, maxMetrics=300001", "children": [ { "duration_msec": 0.504, "message": "search for the first non-negative filter with the smallest cost", "children": [ { "duration_msec": 0.494, "message": "get metric ids for filter and date: filter={__name__=\"node_filesystem_readonly\"}, date=2023-07-05, maxMetrics=15000050, maxLoopsCount=9223372036854775807", "children": [ { "duration_msec": 0.43, "message": "found 4555 metric ids for filter={__name__=\"node_filesystem_readonly\"} using exact search; spent 8833 loops" } ] }, { "duration_msec": 0, "message": "the filter={__name__=\"node_filesystem_readonly\"} matches less than 15000050 series (actually 4555 series); use it" } ] }, { "duration_msec": 0, "message": "intersect the remaining 0 filters with the found 4555 metric ids" }, { "duration_msec": 0, "message": "found 4555 metric ids" } ] } ] } ] }, { "duration_msec": 0.086, "message": "sort 4555 matching metric ids" }, { "duration_msec": 13.961, "message": "load 4555 tsids from 4555 metric ids" } ] }, { "duration_msec": 0.473, "message": "search for tsids in the previous indexdb", "children": [ { "duration_msec": 0.468, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 5746 bytes" }, { "duration_msec": 0.308, "message": "decompressed tsids to 146440 bytes" }, { "duration_msec": 0.145, "message": "unmarshaled 4576 tsids" } ] } ] }, { "duration_msec": 0.812, "message": "merge 4555 tsids from the current indexdb with 4576 tsids from the previous indexdb; result: 4782 tsids" }, { "duration_msec": 2.097, "message": "sort 4782 tsids" }, { "duration_msec": 2.769, "message": "put 4782 tsids in cache", "children": [ { "duration_msec": 0.11, "message": "marshaled 4782 tsids into 153032 bytes" }, { "duration_msec": 0.537, "message": "compressed 4782 tsids into 6008 bytes" }, { "duration_msec": 2.114, "message": "stored 4782 compressed tsids into cache" } ] } ] }, { "duration_msec": 0.231, "message": "prefetch metric names for 4782 tsids", "children": [ { "duration_msec": 0.228, "message": "456 out of 4782 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=456" } ] }, { "duration_msec": 3.908, "message": "search for parts with data for 4782 series" } ] }, { "duration_msec": 145.401, "message": "sent 47724 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 1042.244, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 1041.795, "message": "rpc call search_v7()", "children": [ { "duration_msec": 143.968, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 14.679, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 10.892, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 7.151, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.003, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.002, "message": "cache miss" } ] }, { "duration_msec": 0.562, "message": "search for metric ids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z], maxMetrics=300000", "children": [ { "duration_msec": 0.556, "message": "update metric ids: filters={AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}, timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]: updated 4582 metric ids", "children": [ { "duration_msec": 0.549, "message": "search for metric ids on a particular day: filters={AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}, date=2023-07-05, maxMetrics=300001", "children": [ { "duration_msec": 0.54, "message": "search for the first non-negative filter with the smallest cost", "children": [ { "duration_msec": 0.53, "message": "get metric ids for filter and date: filter={__name__=\"node_filesystem_readonly\"}, date=2023-07-05, maxMetrics=15000050, maxLoopsCount=9223372036854775807", "children": [ { "duration_msec": 0.473, "message": "found 4582 metric ids for filter={__name__=\"node_filesystem_readonly\"} using exact search; spent 8884 loops" } ] }, { "duration_msec": 0, "message": "the filter={__name__=\"node_filesystem_readonly\"} matches less than 15000050 series (actually 4582 series); use it" } ] }, { "duration_msec": 0, "message": "intersect the remaining 0 filters with the found 4582 metric ids" }, { "duration_msec": 0, "message": "found 4582 metric ids" } ] } ] } ] }, { "duration_msec": 0.081, "message": "sort 4582 matching metric ids" }, { "duration_msec": 6.433, "message": "load 4582 tsids from 4582 metric ids" } ] }, { "duration_msec": 0.336, "message": "search for tsids in the previous indexdb", "children": [ { "duration_msec": 0.328, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 5783 bytes" }, { "duration_msec": 0.157, "message": "decompressed tsids to 146856 bytes" }, { "duration_msec": 0.154, "message": "unmarshaled 4589 tsids" } ] } ] }, { "duration_msec": 0.769, "message": "merge 4582 tsids from the current indexdb with 4589 tsids from the previous indexdb; result: 4812 tsids" }, { "duration_msec": 1.795, "message": "sort 4812 tsids" }, { "duration_msec": 0.828, "message": "put 4812 tsids in cache", "children": [ { "duration_msec": 0.117, "message": "marshaled 4812 tsids into 153992 bytes" }, { "duration_msec": 0.581, "message": "compressed 4812 tsids into 6067 bytes" }, { "duration_msec": 0.126, "message": "stored 4812 compressed tsids into cache" } ] } ] }, { "duration_msec": 0.184, "message": "prefetch metric names for 4812 tsids", "children": [ { "duration_msec": 0.181, "message": "477 out of 4812 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=477" } ] }, { "duration_msec": 3.578, "message": "search for parts with data for 4812 series" } ] }, { "duration_msec": 129.226, "message": "sent 42338 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 844.778, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 844.295, "message": "rpc call search_v7()", "children": [ { "duration_msec": 280.505, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.143, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.521, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.518, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.516, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 5988 bytes" }, { "duration_msec": 0.363, "message": "decompressed tsids to 150280 bytes" }, { "duration_msec": 0.139, "message": "unmarshaled 4696 tsids" } ] } ] } ] }, { "duration_msec": 0.202, "message": "prefetch metric names for 4696 tsids", "children": [ { "duration_msec": 0.198, "message": "473 out of 4696 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=473" } ] }, { "duration_msec": 4.395, "message": "search for parts with data for 4696 series" } ] }, { "duration_msec": 275.305, "message": "sent 25971 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 579.039, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 578.534, "message": "rpc call search_v7()", "children": [ { "duration_msec": 138.015, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.575, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.373, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.371, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.368, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.006, "message": "found tsids with compressed size: 5884 bytes" }, { "duration_msec": 0.234, "message": "decompressed tsids to 147848 bytes" }, { "duration_msec": 0.124, "message": "unmarshaled 4620 tsids" } ] } ] } ] }, { "duration_msec": 0.185, "message": "prefetch metric names for 4620 tsids", "children": [ { "duration_msec": 0.181, "message": "459 out of 4620 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=459" } ] }, { "duration_msec": 4.997, "message": "search for parts with data for 4620 series" } ] }, { "duration_msec": 132.394, "message": "sent 32731 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 801.162, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 800.638, "message": "rpc call search_v7()", "children": [ { "duration_msec": 64.668, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 19.033, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.491, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.487, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.485, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 7333 bytes" }, { "duration_msec": 0.287, "message": "decompressed tsids to 148936 bytes" }, { "duration_msec": 0.185, "message": "unmarshaled 4654 tsids" } ] } ] } ] }, { "duration_msec": 0.179, "message": "prefetch metric names for 4654 tsids", "children": [ { "duration_msec": 0.176, "message": "464 out of 4654 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=464" } ] }, { "duration_msec": 18.341, "message": "search for parts with data for 4654 series" } ] }, { "duration_msec": 45.58, "message": "sent 13572 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 1114.495, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 1114.489, "message": "rpc call search_v7()", "children": [ { "duration_msec": 655.284, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 8.843, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 2.421, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_readonly\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 2.416, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 2.414, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 6008 bytes" }, { "duration_msec": 2.045, "message": "decompressed tsids to 151176 bytes" }, { "duration_msec": 0.352, "message": "unmarshaled 4724 tsids" } ] } ] } ] }, { "duration_msec": 0.807, "message": "prefetch metric names for 4724 tsids", "children": [ { "duration_msec": 0.8, "message": "473 out of 4724 metric names must be pre-fetched" }, { "duration_msec": 0.004, "message": "skip pre-fetching metric names for low number of metrid ids=473" } ] }, { "duration_msec": 5.59, "message": "search for parts with data for 4724 series" } ] }, { "duration_msec": 646.379, "message": "sent 47005 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 320.52, "message": "fetch unique series=18000, blocks=247780, samples=3226077, bytes=24756187" } ] }, { "duration_msec": 3735.9, "message": "rollup default_rollup() over 18000 series; rollupConfigs=[timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0, points=1]", "children": [ { "duration_msec": 3729.006, "message": "parallel process of fetched data: series=18000, samples=720000" }, { "duration_msec": 6.787, "message": "samplesScanned=738000" } ] }, { "duration_msec": 0.003, "message": "rollup cache put: query=node_filesystem_readonly, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0, series=18000", "children": [ { "duration_msec": 0, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] }, { "duration_msec": 0.005, "message": "eval: query=0, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] }, { "duration_msec": 5530.576, "message": "eval: query=(((node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host-10|host-100|host-101|host-102|host-10../shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400) < 0), timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=0, points=0, pointsPerSeries=0", "children": [ { "duration_msec": 5530.57, "message": "binary op \"and\": series=0", "children": [ { "duration_msec": 3423.984, "message": "eval: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..1/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400) < 0, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 3423.979, "message": "binary op \"<\": series=6750", "children": [ { "duration_msec": 3233.744, "message": "eval: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..1be11/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400), timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 0.004, "message": "eval: query=86400, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" }, { "duration_msec": 3233.718, "message": "rollup predict_linear(): timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=3600000: neededMemoryBytes=108000", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..1be11/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400), timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=3600000", "children": [ { "duration_msec": 0, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 1633.087, "message": "fetch matching series: accountID=3850, projectID=0, filters=[{__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!~\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 785.531, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 773.376, "message": "rpc call search_v7()", "children": [ { "duration_msec": 28.497, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 2.818, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.146, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.13, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.106, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.01, "message": "found tsids with compressed size: 2457 bytes" }, { "duration_msec": 0.045, "message": "decompressed tsids to 55400 bytes" }, { "duration_msec": 0.049, "message": "unmarshaled 1731 tsids" } ] } ] } ] }, { "duration_msec": 0.064, "message": "prefetch metric names for 1731 tsids", "children": [ { "duration_msec": 0.061, "message": "0 out of 1731 metric names must be pre-fetched" }, { "duration_msec": 0, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 2.575, "message": "search for parts with data for 1731 series" } ] }, { "duration_msec": 25.518, "message": "sent 8655 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 983.687, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 971.474, "message": "rpc call search_v7()", "children": [ { "duration_msec": 98.447, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.153, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.195, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.173, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.167, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.014, "message": "found tsids with compressed size: 2409 bytes" }, { "duration_msec": 0.066, "message": "decompressed tsids to 53672 bytes" }, { "duration_msec": 0.081, "message": "unmarshaled 1677 tsids" } ] } ] } ] }, { "duration_msec": 0.089, "message": "prefetch metric names for 1677 tsids", "children": [ { "duration_msec": 0.086, "message": "0 out of 1677 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 3.835, "message": "search for parts with data for 1677 series" } ] }, { "duration_msec": 94.108, "message": "sent 14098 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 298.691, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 286.445, "message": "rpc call search_v7()", "children": [ { "duration_msec": 58.946, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.262, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.16, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.141, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.13, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.012, "message": "found tsids with compressed size: 2405 bytes" }, { "duration_msec": 0.065, "message": "decompressed tsids to 53576 bytes" }, { "duration_msec": 0.05, "message": "unmarshaled 1674 tsids" } ] } ] } ] }, { "duration_msec": 0.085, "message": "prefetch metric names for 1674 tsids", "children": [ { "duration_msec": 0.082, "message": "0 out of 1674 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.983, "message": "search for parts with data for 1674 series" } ] }, { "duration_msec": 53.491, "message": "sent 22559 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 1063.05, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 1050.775, "message": "rpc call search_v7()", "children": [ { "duration_msec": 97.6, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.426, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.152, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.134, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.122, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2430 bytes" }, { "duration_msec": 0.051, "message": "decompressed tsids to 53352 bytes" }, { "duration_msec": 0.059, "message": "unmarshaled 1667 tsids" } ] } ] } ] }, { "duration_msec": 0.089, "message": "prefetch metric names for 1667 tsids", "children": [ { "duration_msec": 0.086, "message": "0 out of 1667 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.159, "message": "search for parts with data for 1667 series" } ] }, { "duration_msec": 93.047, "message": "sent 20231 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 535.021, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 522.685, "message": "rpc call search_v7()", "children": [ { "duration_msec": 65.255, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.653, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.207, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.185, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.176, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2397 bytes" }, { "duration_msec": 0.08, "message": "decompressed tsids to 52808 bytes" }, { "duration_msec": 0.08, "message": "unmarshaled 1650 tsids" } ] } ] } ] }, { "duration_msec": 0.124, "message": "prefetch metric names for 1650 tsids", "children": [ { "duration_msec": 0.12, "message": "0 out of 1650 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.272, "message": "search for parts with data for 1650 series" } ] }, { "duration_msec": 59.409, "message": "sent 14241 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 872.709, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 860.323, "message": "rpc call search_v7()", "children": [ { "duration_msec": 143.56, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 9.858, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.187, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.17, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.166, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2436 bytes" }, { "duration_msec": 0.073, "message": "decompressed tsids to 54600 bytes" }, { "duration_msec": 0.08, "message": "unmarshaled 1706 tsids" } ] } ] } ] }, { "duration_msec": 0.089, "message": "prefetch metric names for 1706 tsids", "children": [ { "duration_msec": 0.086, "message": "0 out of 1706 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 9.553, "message": "search for parts with data for 1706 series" } ] }, { "duration_msec": 133.582, "message": "sent 16115 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 219.623, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 207.189, "message": "rpc call search_v7()", "children": [ { "duration_msec": 29.818, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.893, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.2, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.179, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.166, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.01, "message": "found tsids with compressed size: 2433 bytes" }, { "duration_msec": 0.082, "message": "decompressed tsids to 54248 bytes" }, { "duration_msec": 0.071, "message": "unmarshaled 1695 tsids" } ] } ] } ] }, { "duration_msec": 0.113, "message": "prefetch metric names for 1695 tsids", "children": [ { "duration_msec": 0.109, "message": "0 out of 1695 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.533, "message": "search for parts with data for 1695 series" } ] }, { "duration_msec": 23.724, "message": "sent 8516 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 549.668, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 549.661, "message": "rpc call search_v7()", "children": [ { "duration_msec": 66.636, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.608, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.176, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype!=\"tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T06:17:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.151, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.138, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2465 bytes" }, { "duration_msec": 0.061, "message": "decompressed tsids to 54408 bytes" }, { "duration_msec": 0.063, "message": "unmarshaled 1700 tsids" } ] } ] } ] }, { "duration_msec": 0.096, "message": "prefetch metric names for 1700 tsids", "children": [ { "duration_msec": 0.093, "message": "0 out of 1700 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.305, "message": "search for parts with data for 1700 series" } ] }, { "duration_msec": 60.864, "message": "sent 19384 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 1079.476, "message": "fetch unique series=6750, blocks=123799, samples=6070973, bytes=19406168" } ] }, { "duration_msec": 1599.858, "message": "rollup predict_linear() over 6750 series; rollupConfigs=[timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=3600000, points=1]", "children": [ { "duration_msec": 1592.17, "message": "parallel process of fetched data: series=6750, samples=1755000" }, { "duration_msec": 7.652, "message": "samplesScanned=3375000" } ] }, { "duration_msec": 0.006, "message": "rollup cache put: query=predict_linear(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", fstype!~\"tmpfs\", instance=~\"host-0|host-1|host-10|host-..1be11/shm|/tmp|/usr/share/oem|/var|/var/lib/cloud|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}[1h], 86400), timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=3600000, series=6750", "children": [ { "duration_msec": 0.001, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] }, { "duration_msec": 0.006, "message": "eval: query=0, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] }, { "duration_msec": 1893.23, "message": "eval: query=((node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|ho../flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}) < 10, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 1893.223, "message": "binary op \"<\": series=6750", "children": [ { "duration_msec": 1485.303, "message": "eval: query=(node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|hos..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 1485.289, "message": "binary op \"/\": series=6750", "children": [ { "duration_msec": 632.379, "message": "eval: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host../flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"} * 100, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 632.373, "message": "binary op \"*\": series=6750", "children": [ { "duration_msec": 571.731, "message": "eval: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 571.722, "message": "rollup default_rollup(): timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0: neededMemoryBytes=108000", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0", "children": [ { "duration_msec": 0, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 431.681, "message": "fetch matching series: accountID=3850, projectID=0, filters=[{__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 37.959, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 37.861, "message": "rpc call search_v7()", "children": [ { "duration_msec": 16.182, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.215, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.235, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.196, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.189, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.01, "message": "found tsids with compressed size: 2457 bytes" }, { "duration_msec": 0.113, "message": "decompressed tsids to 55400 bytes" }, { "duration_msec": 0.062, "message": "unmarshaled 1731 tsids" } ] } ] } ] }, { "duration_msec": 0.097, "message": "prefetch metric names for 1731 tsids", "children": [ { "duration_msec": 0.095, "message": "0 out of 1731 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.848, "message": "search for parts with data for 1731 series" } ] }, { "duration_msec": 10.8, "message": "sent 3462 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 107.043, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 106.894, "message": "rpc call search_v7()", "children": [ { "duration_msec": 44.132, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 6.036, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.225, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.175, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.171, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2409 bytes" }, { "duration_msec": 0.086, "message": "decompressed tsids to 53672 bytes" }, { "duration_msec": 0.072, "message": "unmarshaled 1677 tsids" } ] } ] } ] }, { "duration_msec": 0.101, "message": "prefetch metric names for 1677 tsids", "children": [ { "duration_msec": 0.098, "message": "0 out of 1677 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.675, "message": "search for parts with data for 1677 series" } ] }, { "duration_msec": 37.916, "message": "sent 10744 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 229.909, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 229.727, "message": "rpc call search_v7()", "children": [ { "duration_msec": 175.975, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.196, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.206, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.18, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.173, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2405 bytes" }, { "duration_msec": 0.085, "message": "decompressed tsids to 53576 bytes" }, { "duration_msec": 0.072, "message": "unmarshaled 1674 tsids" } ] } ] } ] }, { "duration_msec": 0.118, "message": "prefetch metric names for 1674 tsids", "children": [ { "duration_msec": 0.114, "message": "0 out of 1674 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.834, "message": "search for parts with data for 1674 series" } ] }, { "duration_msec": 170.65, "message": "sent 17537 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 145.693, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 145.491, "message": "rpc call search_v7()", "children": [ { "duration_msec": 55.427, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 13.256, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.318, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.252, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.247, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2430 bytes" }, { "duration_msec": 0.164, "message": "decompressed tsids to 53352 bytes" }, { "duration_msec": 0.07, "message": "unmarshaled 1667 tsids" } ] } ] } ] }, { "duration_msec": 0.13, "message": "prefetch metric names for 1667 tsids", "children": [ { "duration_msec": 0.126, "message": "0 out of 1667 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 12.767, "message": "search for parts with data for 1667 series" } ] }, { "duration_msec": 41.857, "message": "sent 3334 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 190.552, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 190.274, "message": "rpc call search_v7()", "children": [ { "duration_msec": 63.617, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 6.649, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.279, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.227, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.216, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2397 bytes" }, { "duration_msec": 0.149, "message": "decompressed tsids to 52808 bytes" }, { "duration_msec": 0.055, "message": "unmarshaled 1650 tsids" } ] } ] } ] }, { "duration_msec": 0.104, "message": "prefetch metric names for 1650 tsids", "children": [ { "duration_msec": 0.101, "message": "0 out of 1650 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 6.23, "message": "search for parts with data for 1650 series" } ] }, { "duration_msec": 56.828, "message": "sent 9291 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 167.016, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 166.7, "message": "rpc call search_v7()", "children": [ { "duration_msec": 76.839, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.863, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.208, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.182, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.177, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2436 bytes" }, { "duration_msec": 0.111, "message": "decompressed tsids to 54600 bytes" }, { "duration_msec": 0.05, "message": "unmarshaled 1706 tsids" } ] } ] } ] }, { "duration_msec": 0.102, "message": "prefetch metric names for 1706 tsids", "children": [ { "duration_msec": 0.099, "message": "0 out of 1706 metric names must be pre-fetched" }, { "duration_msec": 0.002, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.514, "message": "search for parts with data for 1706 series" } ] }, { "duration_msec": 71.828, "message": "sent 12703 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 144.193, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 143.838, "message": "rpc call search_v7()", "children": [ { "duration_msec": 46.74, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.983, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.195, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.168, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.162, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2433 bytes" }, { "duration_msec": 0.092, "message": "decompressed tsids to 54248 bytes" }, { "duration_msec": 0.055, "message": "unmarshaled 1695 tsids" } ] } ] } ] }, { "duration_msec": 0.079, "message": "prefetch metric names for 1695 tsids", "children": [ { "duration_msec": 0.077, "message": "0 out of 1695 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.673, "message": "search for parts with data for 1695 series" } ] }, { "duration_msec": 41.608, "message": "sent 5126 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 401.978, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 401.948, "message": "rpc call search_v7()", "children": [ { "duration_msec": 376.097, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 6.327, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.751, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_avail_bytes\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.727, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.717, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.009, "message": "found tsids with compressed size: 2465 bytes" }, { "duration_msec": 0.09, "message": "decompressed tsids to 54408 bytes" }, { "duration_msec": 0.611, "message": "unmarshaled 1700 tsids" } ] } ] } ] }, { "duration_msec": 0.084, "message": "prefetch metric names for 1700 tsids", "children": [ { "duration_msec": 0.081, "message": "0 out of 1700 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.447, "message": "search for parts with data for 1700 series" } ] }, { "duration_msec": 369.628, "message": "sent 17684 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 22.68, "message": "fetch unique series=6750, blocks=79881, samples=1227641, bytes=10238740" } ] }, { "duration_msec": 139.336, "message": "rollup default_rollup() over 6750 series; rollupConfigs=[timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0, points=1]", "children": [ { "duration_msec": 136.884, "message": "parallel process of fetched data: series=6750, samples=270000" }, { "duration_msec": 2.421, "message": "samplesScanned=276750" } ] }, { "duration_msec": 0.002, "message": "rollup cache put: query=node_filesystem_avail_bytes{device=~\"/dev/root|/dev/sda1|/dev/sda8\", device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\", instance=~\"host-0|host-1|host..rnetes/flexvolume|/mnt/stateful_partition|/usr/share/oem|/var|/var/lib/containerd|/var/lib/docker|/var/lib/google|/var/lib/kubelet|/var/lib/toolbox\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0, series=6750", "children": [ { "duration_msec": 0, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] }, { "duration_msec": 0.009, "message": "eval: query=100, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] }, { "duration_msec": 299.587, "message": "eval: query=node_filesystem_size_bytes{cluster=\"stag-benchmark-gcp-us-west1-929c8c20\", cluster_id=\"large-db\", component=\"node_exporter\", device=~\"/dev/root|/dev/..der=\"gcp\", seed_region=\"us-west1\", shoot_infra=\"gke\", shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\", tenant=\"benchmark\", tidb_cluster_id=\"large\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=6750, points=6750, pointsPerSeries=1", "children": [ { "duration_msec": 299.569, "message": "rollup default_rollup(): timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0: neededMemoryBytes=108000", "children": [ { "duration_msec": 0.002, "message": "rollup cache get: query=node_filesystem_size_bytes{cluster=\"stag-benchmark-gcp-us-west1-929c8c20\", cluster_id=\"large-db\", component=\"node_exporter\", device=~\"/dev/root|/dev/..der=\"gcp\", seed_region=\"us-west1\", shoot_infra=\"gke\", shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\", tenant=\"benchmark\", tidb_cluster_id=\"large\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0", "children": [ { "duration_msec": 0.001, "message": "do not fetch series from cache, since it is disabled in the current context" } ] }, { "duration_msec": 176.738, "message": "fetch matching series: accountID=3850, projectID=0, filters=[{__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 38.002, "message": "rpc at vmstorage vmstorage-vmcluster-0.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 37.859, "message": "rpc call search_v7()", "children": [ { "duration_msec": 18.606, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.861, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.244, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.202, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.195, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.026, "message": "found tsids with compressed size: 2380 bytes" }, { "duration_msec": 0.123, "message": "decompressed tsids to 52872 bytes" }, { "duration_msec": 0.044, "message": "unmarshaled 1652 tsids" } ] } ] } ] }, { "duration_msec": 0.07, "message": "prefetch metric names for 1652 tsids", "children": [ { "duration_msec": 0.068, "message": "5 out of 1652 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=5" } ] }, { "duration_msec": 4.454, "message": "search for parts with data for 1652 series" } ] }, { "duration_msec": 13.457, "message": "sent 3332 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 124.101, "message": "rpc at vmstorage vmstorage-vmcluster-1.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 124.076, "message": "rpc call search_v7()", "children": [ { "duration_msec": 93.393, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 18.886, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.264, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.193, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.185, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.015, "message": "found tsids with compressed size: 2375 bytes" }, { "duration_msec": 0.086, "message": "decompressed tsids to 52936 bytes" }, { "duration_msec": 0.081, "message": "unmarshaled 1654 tsids" } ] } ] } ] }, { "duration_msec": 0.099, "message": "prefetch metric names for 1654 tsids", "children": [ { "duration_msec": 0.096, "message": "0 out of 1654 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 18.434, "message": "search for parts with data for 1654 series" } ] }, { "duration_msec": 74.183, "message": "sent 10764 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 132.844, "message": "rpc at vmstorage vmstorage-vmcluster-2.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 132.737, "message": "rpc call search_v7()", "children": [ { "duration_msec": 41.314, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.436, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.201, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.158, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.147, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2373 bytes" }, { "duration_msec": 0.065, "message": "decompressed tsids to 52936 bytes" }, { "duration_msec": 0.068, "message": "unmarshaled 1654 tsids" } ] } ] } ] }, { "duration_msec": 0.082, "message": "prefetch metric names for 1654 tsids", "children": [ { "duration_msec": 0.079, "message": "0 out of 1654 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.11, "message": "search for parts with data for 1654 series" } ] }, { "duration_msec": 35.637, "message": "sent 17383 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 45.568, "message": "rpc at vmstorage vmstorage-vmcluster-3.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 45.431, "message": "rpc call search_v7()", "children": [ { "duration_msec": 16.436, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.37, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.189, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.145, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.138, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2419 bytes" }, { "duration_msec": 0.062, "message": "decompressed tsids to 53864 bytes" }, { "duration_msec": 0.061, "message": "unmarshaled 1683 tsids" } ] } ] } ] }, { "duration_msec": 0.086, "message": "prefetch metric names for 1683 tsids", "children": [ { "duration_msec": 0.083, "message": "0 out of 1683 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.036, "message": "search for parts with data for 1683 series" } ] }, { "duration_msec": 11.827, "message": "sent 3366 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 90.236, "message": "rpc at vmstorage vmstorage-vmcluster-4.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 90.023, "message": "rpc call search_v7()", "children": [ { "duration_msec": 84.252, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 5.259, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.196, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.159, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.152, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.012, "message": "found tsids with compressed size: 2455 bytes" }, { "duration_msec": 0.06, "message": "decompressed tsids to 54888 bytes" }, { "duration_msec": 0.076, "message": "unmarshaled 1715 tsids" } ] } ] } ] }, { "duration_msec": 0.09, "message": "prefetch metric names for 1715 tsids", "children": [ { "duration_msec": 0.087, "message": "0 out of 1715 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 4.875, "message": "search for parts with data for 1715 series" } ] }, { "duration_msec": 78.765, "message": "sent 9979 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 109.317, "message": "rpc at vmstorage vmstorage-vmcluster-5.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 109.076, "message": "rpc call search_v7()", "children": [ { "duration_msec": 48.446, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 6.355, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.216, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.161, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.154, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2495 bytes" }, { "duration_msec": 0.075, "message": "decompressed tsids to 56360 bytes" }, { "duration_msec": 0.064, "message": "unmarshaled 1761 tsids" } ] } ] } ] }, { "duration_msec": 0.103, "message": "prefetch metric names for 1761 tsids", "children": [ { "duration_msec": 0.1, "message": "0 out of 1761 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 5.967, "message": "search for parts with data for 1761 series" } ] }, { "duration_msec": 41.842, "message": "sent 13101 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 27.794, "message": "rpc at vmstorage vmstorage-vmcluster-6.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 27.521, "message": "rpc call search_v7()", "children": [ { "duration_msec": 21.069, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 4.377, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.271, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.222, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.214, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.057, "message": "found tsids with compressed size: 2473 bytes" }, { "duration_msec": 0.102, "message": "decompressed tsids to 55336 bytes" }, { "duration_msec": 0.051, "message": "unmarshaled 1729 tsids" } ] } ] } ] }, { "duration_msec": 0.085, "message": "prefetch metric names for 1729 tsids", "children": [ { "duration_msec": 0.082, "message": "0 out of 1729 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 3.972, "message": "search for parts with data for 1729 series" } ] }, { "duration_msec": 16.509, "message": "sent 5345 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 152.146, "message": "rpc at vmstorage vmstorage-vmcluster-7.vmstorage-vmcluster.observability:8401", "children": [ { "duration_msec": 152.123, "message": "rpc call search_v7()", "children": [ { "duration_msec": 118.338, "message": "rpc call search_v7() at vmstorage", "children": [ { "duration_msec": 24.015, "message": "init series search: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.189, "message": "search for matching tsids: filters=[{AccountID=3850,ProjectID=0,__name__=\"node_filesystem_size_bytes\",cluster=\"stag-benchmark-gcp-us-west1-929c8c20\",cluster_id=\"large-db\",component=\"node_exporter\",device=~\"/dev/root|/dev/sda1|/dev/sda8\",device=~\"/dev/root|/dev/sda1|/dev/sda8|shm|tmpfs\",fstype=~\"ext2|ext4\",instance=~\"host-0|host-1|host-10|host-10..st-96|host-97|host-98|host-99\",job=\"node_exporter\",mountpoint=~\"/|/etc/machine-id|/home|/home../lib/kubelet|/var/lib/toolbox\",mountpoint=~\"/|/home|/home/containerd|/hom../lib/kubelet|/var/lib/toolbox\",project=\"123456\",revision=~\"r0|r4|r5\",seed_provider=\"gcp\",seed_region=\"us-west1\",shoot_infra=\"gke\",shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\",tenant=\"benchmark\",tidb_cluster_id=\"large\"}], timeRange=[2023-07-05T07:12:45.23Z..2023-07-05T07:22:45.23Z]", "children": [ { "duration_msec": 0.141, "message": "search for tsids in the current indexdb", "children": [ { "duration_msec": 0.134, "message": "search for tsids in tag filters cache", "children": [ { "duration_msec": 0.011, "message": "found tsids with compressed size: 2391 bytes" }, { "duration_msec": 0.064, "message": "decompressed tsids to 52872 bytes" }, { "duration_msec": 0.056, "message": "unmarshaled 1652 tsids" } ] } ] } ] }, { "duration_msec": 0.082, "message": "prefetch metric names for 1652 tsids", "children": [ { "duration_msec": 0.079, "message": "0 out of 1652 metric names must be pre-fetched" }, { "duration_msec": 0.001, "message": "skip pre-fetching metric names for low number of metrid ids=0" } ] }, { "duration_msec": 23.684, "message": "search for parts with data for 1652 series" } ] }, { "duration_msec": 94.07, "message": "sent 17095 blocks to vmselect" } ] } ] } ] }, { "duration_msec": 20.573, "message": "fetch unique series=6750, blocks=80365, samples=1227821, bytes=8132190" } ] }, { "duration_msec": 121.673, "message": "rollup default_rollup() over 6750 series; rollupConfigs=[timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0, points=1]", "children": [ { "duration_msec": 120.035, "message": "parallel process of fetched data: series=6750, samples=270000" }, { "duration_msec": 1.6, "message": "samplesScanned=276750" } ] }, { "duration_msec": 0.007, "message": "rollup cache put: query=node_filesystem_size_bytes{cluster=\"stag-benchmark-gcp-us-west1-929c8c20\", cluster_id=\"large-db\", component=\"node_exporter\", device=~\"/dev/root|/dev/..der=\"gcp\", seed_region=\"us-west1\", shoot_infra=\"gke\", shoot_name=\"stag-benchmark-gcp-us-west1-929c8c20\", tenant=\"benchmark\", tidb_cluster_id=\"large\"}, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, window=0, series=6750", "children": [ { "duration_msec": 0.001, "message": "do not store series to cache, since it is disabled in the current context" } ] } ] } ] } ] } ] }, { "duration_msec": 0.008, "message": "eval: query=10, timeRange=[2023-07-05T07:22:45.23Z..2023-07-05T07:22:45.23Z], step=300000, mayCache=false: series=1, points=1, pointsPerSeries=1" } ] } ] } ] } ] } ] } ] }, { "duration_msec": 0.003, "message": "leave only the first point in every series" }, { "duration_msec": 0.007, "message": "sort series by metric name and labels" }, { "duration_msec": 0.012, "message": "generate /api/v1/query response for series=0" } ] }