import os import sys sys.path.append(os.path.dirname(__file__)) from common import ( DATASOURCE, DATASOURCE_INPUT, OPTIONAL_QUANTILE, OPTIONAL_QUANTILE_INPUT, Layout, expr_avg, expr_count_rate, expr_histogram_avg, expr_histogram_quantile, expr_max, expr_max_rate, expr_min, expr_operator, expr_simple, expr_sum, expr_sum_aggr_over_time, expr_sum_delta, expr_sum_increase, expr_sum_rate, expr_topk, graph_legend, graph_panel, graph_panel_histogram_quantiles, heatmap_panel, heatmap_panel_graph_panel_histogram_quantile_pairs, series_override, stat_panel, target, template, yaxes, yaxis, ) from grafanalib import formatunits as UNITS from grafanalib.core import ( GRAPH_TOOLTIP_MODE_SHARED_CROSSHAIR, HIDE_VARIABLE, NULL_AS_NULL, SHOW, Dashboard, GraphThreshold, Panel, RowPanel, SeriesOverride, StatValueMappingItem, StatValueMappings, Templating, ) #### Metrics Definition Start #### class MinimalSeriesOverride: def __init__(self, alias: str, color: str): self.alias = alias self.color = color def to_json_data(self): return { "alias": self.alias, "color": self.color, } def stacked_thread_cpu_panel( title: str, label_selector: str, description=None, by_labels: list[str] = ["instance"], thresholds: list[GraphThreshold] | None = None, additional_groupby: bool = False, metric_namespace: str = "tikv", ) -> Panel: base_legend = " ".join([f"{{{{{b}}}}}" for b in by_labels]).strip() all_labels = by_labels + ["kind"] legend_format = f"{base_legend} {{{{kind}}}}" if base_legend else "{{{{kind}}}}" targets = [ target( expr=expr_sum_rate( f"{metric_namespace}_thread_cpu_detail_seconds_total", label_selectors=[label_selector], by_labels=all_labels, ), legend_format=legend_format, additional_groupby=additional_groupby, ), ] overrides = [ MinimalSeriesOverride(alias="/ user$/", color="#2ca02c"), MinimalSeriesOverride(alias="/ sys$/", color="#f2c12e"), MinimalSeriesOverride(alias="/ run_queue_wait$/", color="#ff8c00"), MinimalSeriesOverride(alias="/ blkio_wait$/", color="#d62728"), ] return graph_panel( title=title, description=description, yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=targets, stack=True, line_width=1, fill=5, fill_gradient=0, series_overrides=overrides, thresholds=thresholds or [], ) def total_thread_cpu_panel( title: str, label_selector: str, description=None, by_labels: list[str] = ["instance"], thresholds: list[GraphThreshold] | None = None, additional_groupby: bool = False, metric_namespace: str = "tikv", ) -> Panel: legend = " ".join([f"{{{{{b}}}}}" for b in by_labels]).strip() return graph_panel( title=title, description=description, yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( f"{metric_namespace}_thread_cpu_seconds_total", label_selectors=[label_selector], by_labels=by_labels, ), legend_format=legend if legend else None, additional_groupby=additional_groupby, ) ], stack=False, line_width=1, fill=0, fill_gradient=0, thresholds=thresholds or [], ) def thread_cpu_panel_pair( title: str, label_selector: str, description=None, by_labels: list[str] = ["instance"], thresholds: list[GraphThreshold] | None = None, additional_groupby: bool = False, metric_namespace: str = "tikv", ) -> list[Panel]: return [ total_thread_cpu_panel( title=title, label_selector=label_selector, description=description, by_labels=by_labels, thresholds=thresholds, additional_groupby=additional_groupby, metric_namespace=metric_namespace, ), stacked_thread_cpu_panel( title=f"{title} (by kind)", label_selector=label_selector, description=description, by_labels=by_labels, thresholds=thresholds, additional_groupby=additional_groupby, metric_namespace=metric_namespace, ), ] def Templates() -> Templating: return Templating( list=[ template( name="k8s_cluster", type="query", query="label_values(tikv_engine_size_bytes, k8s_cluster)", data_source=DATASOURCE, hide=HIDE_VARIABLE, ), template( name="tidb_cluster", type="query", query='label_values(tikv_engine_size_bytes{k8s_cluster="$k8s_cluster"}, tidb_cluster)', data_source=DATASOURCE, hide=HIDE_VARIABLE, ), template( name="db", type="query", query='label_values(tikv_engine_size_bytes{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster"}, db)', data_source=DATASOURCE, hide=SHOW, multi=True, include_all=True, ), template( name="command", type="query", query='query_result(tikv_storage_command_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster"}!=0)', data_source=DATASOURCE, hide=SHOW, regex='/\\btype="([^"]+)"/', multi=True, include_all=True, ), template( name="instance", type="query", query='label_values(tikv_engine_size_bytes{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster"}, instance)', data_source=DATASOURCE, hide=SHOW, multi=True, include_all=True, all_value=".*", ), template( name="worker_instance", type="query", query='label_values(tikv_worker_cpu_cores_quota{k8s_cluster ="$k8s_cluster", tidb_cluster="$tidb_cluster"}, instance)', data_source=DATASOURCE, hide=SHOW, multi=True, include_all=True, all_value=".*", ), template( name="titan_db", type="query", query='label_values(tikv_engine_titandb_num_live_blob_file{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster"}, db)', data_source=DATASOURCE, hide=HIDE_VARIABLE, multi=True, include_all=True, ), template( name="additional_groupby", type="custom", query="none,instance", data_source=DATASOURCE, hide=SHOW, ), template( name=OPTIONAL_QUANTILE, type="custom", query="0.99,0.999,0.9999,0.99999,1", data_source=DATASOURCE, hide=SHOW, ), template( name="keyspace_id", type="query", query='label_values({__name__=~"tikv_cpu_token_wait_duration_seconds_count|tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count", k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster"}, keyspace_id)', data_source=DATASOURCE, hide=SHOW, multi=True, include_all=True, ), ] ) def Duration() -> RowPanel: layout = Layout(title="Duration") layout.row( [ graph_panel( title="Write Pipeline Duration", description="Write Pipeline Composition", yaxes=yaxes(left_format=UNITS.SECONDS), lines=False, stack=True, targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_append_log_duration_seconds", is_optional_quantile=True, ), legend_format="Write Raft Log-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_request_wait_time_duration_secs", is_optional_quantile=True, ), legend_format="Propose Wait-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_apply_wait_time_duration_secs", is_optional_quantile=True, ), legend_format="Apply Wait-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_commit_log_duration_seconds", is_optional_quantile=True, ), legend_format="Replicate Raft Log-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_apply_log_duration_seconds", is_optional_quantile=True, ), legend_format="Apply Duration-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), graph_panel( title="Cop Read Duration", description="Read Duration Composition", yaxes=yaxes(left_format=UNITS.SECONDS), lines=False, stack=True, targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_storage_engine_async_request_duration_seconds", ['type="snapshot"'], is_optional_quantile=True, ), legend_format="Get Snapshot-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "tikv_coprocessor_request_wait_seconds", ['type="all"'], is_optional_quantile=True, ), legend_format="Cop Wait-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_coprocessor_request_handle_seconds", is_optional_quantile=True, ), legend_format="Cop Handle-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) return layout.row_panel def Cluster( resource_label_selector: str = 'job=~".*(tikv|copr|schema-manager).*"', ) -> RowPanel: layout = Layout(title="Cluster") layout.row( [ graph_panel( title="Store size", description=( "Approximate durable local used size per TiKV instance " "(row SST catalog + raft)." ), yaxes=yaxes(left_format=UNITS.BYTES_IEC), fill=1, fill_gradient=1, stack=True, legend=graph_legend(max=False), targets=[ target( expr=expr_sum( "tikv_store_size_bytes", label_selectors=['type="all_used"'], ), ), ], ), graph_panel( title="Available size", description="The available capacity size of each TiKV instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), fill=1, fill_gradient=1, stack=True, legend=graph_legend(max=False), targets=[ target( expr=expr_sum( "tikv_store_size_bytes", label_selectors=['type="available"'], ), ), ], ), graph_panel( title="Capacity size", description="The capacity size per TiKV instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), fill=1, fill_gradient=1, stack=True, legend=graph_legend(max=False), targets=[ target( expr=expr_sum( "tikv_store_size_bytes", label_selectors=['type="capacity"'], ), ), ], ), ] ) layout.row( [ graph_panel( title="CPU", description="The CPU usage of each TiKV instance", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( "process_cpu_seconds_total", label_selectors=[resource_label_selector], ), ), target( expr=expr_simple( "tikv_server_cpu_cores_quota", label_selectors=[resource_label_selector], ), legend_format=r"quota-{{instance}}", hide=True, ), target( expr=expr_max( "tikv_server_cpu_cores_quota", label_selectors=[resource_label_selector], by_labels=[], ), legend_format=r"max-quota", ), target( expr=expr_min( "tikv_server_cpu_cores_quota", label_selectors=[resource_label_selector], by_labels=[], ), legend_format=r"min-quota", ), ], ), graph_panel( title="Memory", description="The memory usage per TiKV instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "process_resident_memory_bytes", label_selectors=[resource_label_selector], ), ), target( expr=expr_simple( "tikv_server_memory_quota_bytes", label_selectors=[resource_label_selector], ), legend_format=r"quota-{{instance}}", hide=True, ), target( expr=expr_max( "tikv_server_memory_quota_bytes", label_selectors=[resource_label_selector], by_labels=[], ), legend_format=r"max-quota", ), target( expr=expr_min( "tikv_server_memory_quota_bytes", label_selectors=[resource_label_selector], by_labels=[], ), legend_format=r"min-quota", ), ], ), ] ) layout.row( [ graph_panel( title="IO utilization", description="The I/O utilization per TiKV instance", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( "node_disk_io_time_seconds_total", ), legend_format=r"{{instance}}-{{device}}", ), ], ), graph_panel( title="MBps", description="The total bytes of read and write in each TiKV instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_engine_flow_bytes", label_selectors=['type="wal_file_bytes"'], ), legend_format=r"{{instance}}-write", ), target( expr=expr_sum_rate( "tikv_engine_flow_bytes", label_selectors=['type=~"bytes_read|iter_bytes_read"'], ), legend_format=r"{{instance}}-read", ), target( expr=expr_sum_rate( "tikv_in_memory_engine_flow", label_selectors=['type=~"bytes_read|iter_bytes_read"'], ), legend_format=r"{{instance}}-in-memory-engine-read", ), ], ), ] ) layout.row( [ graph_panel( title="QPS", description="The number of leaders on each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_grpc_msg_duration_seconds_count", label_selectors=['type!="kv_gc"'], by_labels=["instance", "type"], ), legend_format=r"{{instance}}-{{type}}", ), ], ), graph_panel( title="Errps", description="The total number of the gRPC message failures", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_grpc_msg_fail_total", label_selectors=['type!="kv_gc"'], ), legend_format=r"{{instance}}-grpc-msg-fail", ), target( expr=expr_sum_delta( "tikv_pd_heartbeat_message_total", label_selectors=['type="noop"'], ).extra(extra_expr="< 1"), legend_format=r"{{instance}}-pd-heartbeat", ), target( expr=expr_sum_rate( "tikv_critical_error_total", by_labels=["instance", "type"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Leader", description="The number of leaders on each TiKV instance", targets=[ target( expr=expr_sum( "tikv_raftstore_region_count", label_selectors=['type="leader"'], ), ), ], ), graph_panel( title="Region", description="The number of Regions and Buckets on each TiKV instance", targets=[ target( expr=expr_sum( "tikv_raftstore_region_count", label_selectors=['type="region"'], ), ), target( expr=expr_sum( "tikv_raftstore_region_count", label_selectors=['type="buckets"'], ), legend_format=r"{{instance}}-buckets", hide=True, ), ], ), ] ) layout.row( [ graph_panel( title="Leader data size", description=( "Physical sizes summed over region-leader shards " "(unique when summed across stores). " ), yaxes=yaxes(left_format=UNITS.BYTES_IEC), fill=1, fill_gradient=1, legend=graph_legend(max=False), targets=[ target( # row_size (all peers on this store, hidden by default) # is included in store used_size (all_used). expr=expr_sum( "tikv_store_size_bytes", label_selectors=['type="row_size"'], ), legend_format=r"{{instance}}-row", hide=True, ), target( expr=expr_sum( "tikv_store_size_bytes", label_selectors=['type=~"leader_.*_size"'], by_labels=["instance", "type"], ), legend_format=r"{{instance}}-{{type}}", ), ], ), graph_panel( title="Uptime", description="TiKV uptime since the last restart", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_operator( "time()", "-", expr_simple( "process_start_time_seconds", label_selectors=[resource_label_selector], ), ), legend_format=r"{{instance}}", ), ], ), ] ) return layout.row_panel def Errors() -> RowPanel: layout = Layout(title="Errors") layout.row( [ graph_panel( title="Critical error", targets=[ target( expr=expr_sum_rate( "tikv_critical_error_total", by_labels=["instance", "type"], ), ), ], thresholds=[GraphThreshold(value=0.0)], ) ] ) layout.row( [ graph_panel( title="Server is busy", description=""" Shows ServerIsBusy responses constructed for clients and subsystem pressure signals that can make a TiKV instance temporarily unavailable.""", targets=[ target( expr=expr_sum_rate( "tikv_server_is_busy_total", by_labels=["instance", "type"], ), legend_format=r"server-is-busy-{{instance}}-{{type}}", ), target( expr=expr_sum_rate( "tikv_scheduler_too_busy_total", by_labels=["instance", "type"], ), legend_format=r"scheduler-{{instance}}-{{type}}", ), target( expr=expr_sum_rate( "tikv_unified_read_pool_rejected_tasks_total", by_labels=["instance", "name", "reason"], ), legend_format=r"read-pool-{{instance}}-{{name}}-{{reason}}", ), target( expr=expr_sum_rate( "tikv_channel_full_total", by_labels=["instance", "type"], ), legend_format=r"channelfull-{{instance}}-{{type}}", ), target( expr=expr_sum_rate( "tikv_coprocessor_request_error", label_selectors=[ 'reason=~"deadline_exceeded|max_pending_tasks_exceeded|cpu_throttled|memory_quota_exceeded|server_is_busy"' ], by_labels=["instance", "reason"], ), legend_format=r"coprocessor-{{instance}}-{{reason}}", ), target( expr=expr_avg( "tikv_engine_write_stall", label_selectors=[ 'type="write_stall_percentile99"', 'db=~"$db"', ], by_labels=["instance", "db"], ), legend_format=r"stall-{{instance}}-{{db}}", ), target( expr=expr_sum_rate( "tikv_raftstore_store_write_msg_block_wait_duration_seconds_count", ), legend_format=r"store-write-channelfull-{{instance}}", ), target( expr=expr_sum( "tikv_raftstore_process_busy", by_labels=["instance", "type"], ), legend_format=r"{{instance}}-{{type}}", ), ], ), graph_panel( title="Server report failures", description="The total number of reporting failure messages", targets=[ target( expr=expr_sum_rate( "tikv_server_report_failure_msg_total", by_labels=["type", "instance", "store_id"], ), legend_format=r"{{instance}}-{{type}}-to-{{store_id}}", ), ], ), ] ) layout.row( [ graph_panel( title="Raftstore error", description="The number of different raftstore errors on each TiKV instance", targets=[ target( expr=expr_sum_rate( "tikv_storage_engine_async_request_total", label_selectors=['status!~"success|all"'], by_labels=["instance", "status"], ), ), ], ), graph_panel( title="Scheduler error", description="The number of scheduler errors per type on each TiKV instance", targets=[ target( expr=expr_sum_rate( "tikv_scheduler_stage_total", label_selectors=['stage=~"snapshot_err|prepare_write_err"'], by_labels=["instance", "stage"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Coprocessor error", description="The number of different coprocessor errors on each TiKV instance", targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_request_error", by_labels=["instance", "reason"], ), ), ], ), graph_panel( title="gRPC message error", description="The number of gRPC message errors per type on each TiKV instance", targets=[ target( expr=expr_sum_rate( "tikv_grpc_msg_fail_total", by_labels=["instance", "type"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Leader drop", description="The count of dropped leaders per TiKV instance", targets=[ target( expr=expr_sum_delta( "tikv_raftstore_region_count", label_selectors=['type="leader"'], ), ), ], ), graph_panel( title="Leader missing", description="The count of missing leaders per TiKV instance", targets=[ target( expr=expr_sum( "tikv_raftstore_leader_missing", ), ), ], ), ] ) layout.row( [ graph_panel( title="Advertised Address Probe Failures", description=( "The count of advertised address probe failures per TiKV instance, " "grouped by loopback, unresolved, and timeout reasons" ), targets=[ target( expr=expr_simple( "tikv_server_advertise_addr_probe_failure_total" ), legend_format=r"{{instance}}-{{endpoint}}-{{reason}}", ), ], ), graph_panel( title="Log Replication Rejected", description="The count of Log Replication Reject caused by follower memory insufficient", targets=[ target( expr=expr_sum_rate( "tikv_server_raft_append_rejects", ), ), ], ), ] ) return layout.row_panel def Server() -> RowPanel: layout = Layout(title="Server") layout.row( [ graph_panel( title="CF size", description="The size of each column family", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_engine_size_bytes", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Channel full", description="The total number of channel full errors on each TiKV instance", targets=[ target( expr=expr_sum_rate( "tikv_channel_full_total", by_labels=["instance", "type"] ), ), ], ), ] ) layout.row( [ graph_panel( title="Active written leaders", description="The number of leaders being written on each TiKV instance", targets=[ target( expr=expr_sum_rate( "tikv_region_written_keys_count", ), ), ], ), ] ) layout.row( [ heatmap_panel( title="Approximate region size", metric="tikv_raftstore_region_size_bucket", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), ), graph_panel_histogram_quantiles( title="Approximate region size", description="The approximate Region size", metric="tikv_raftstore_region_size", yaxes=yaxes(left_format=UNITS.BYTES_IEC), hide_count=True, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Huge memtable size", description="Histogram of huge mem table bytes for regions", metric="kv_engine_region_huge_mem_table_bytes", yaxes=yaxes(left_format=UNITS.BYTES_IEC), hide_count=True, ), graph_panel_histogram_quantiles( title="Huge l0 table size", description="Histogram of huge l0 table bytes for regions", metric="kv_engine_region_huge_l0_table_bytes", yaxes=yaxes(left_format=UNITS.BYTES_IEC), hide_count=True, ), ] ) layout.row( [ heatmap_panel( title="Region written bytes", metric="tikv_region_written_bytes_bucket", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), ), graph_panel( title="Region average written bytes", description="The average rate of writing bytes to Regions per TiKV instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_histogram_avg("tikv_region_written_bytes"), legend_format="{{instance}}", ), ], ), ] ) layout.row( [ heatmap_panel( title="Region written keys", metric="tikv_region_written_keys_bucket", ), graph_panel( title="Region average written keys", description="The average rate of written keys to Regions per TiKV instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_histogram_avg("tikv_region_written_keys"), legend_format="{{instance}}", ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Clear overlap region duration", description="Bucketed histogram of clear overlap region duration.", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_clear_overlap_region_duration_seconds", by_labels=["type"], hide_count=True, ), ] ) layout.row( [ heatmap_panel( title="Written key size", description="The key size for every put of apply worker", metric="tikv_raftstore_apply_key_size_bucket", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), ), heatmap_panel( title="Written value size for every put of apply worker", metric="tikv_raftstore_apply_value_size_bucket", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), ), ] ) layout.row( [ graph_panel( title="Hibernate Peers", description="The number of peers in hibernated state", targets=[ target( expr=expr_sum( "tikv_raftstore_hibernated_peer_state", by_labels=["instance", "state"], ), ), ], ), graph_panel( title="Memory trace", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_simple( "tikv_server_mem_trace_sum", label_selectors=['name=~"raftstore-.*"'], ), legend_format="{{instance}}-{{name}}", ), target( expr=expr_simple( "raft_engine_memory_usage", ), legend_format="{{instance}}-raft-engine", ), ], ), ] ) layout.row( [ graph_panel( title="Raft Entry Cache Evicts", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_raft_entries_evict_bytes", ), ), ], ), graph_panel( title="Resolve address duration", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_server_address_resolve_duration_secs", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-" + OPTIONAL_QUANTILE_INPUT, ), ], ), ] ) layout.row( [ graph_panel( title="Thread Pool Schedule Wait Duration" + OPTIONAL_QUANTILE_INPUT, yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_yatp_pool_schedule_wait_duration", by_labels=["name", "priority"], is_optional_quantile=True, ), legend_format="{{name}}-{{priority}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], thresholds=[GraphThreshold(value=1.0)], ), graph_panel( title="Average Thread Pool Schedule Wait Duration", description="The average schedule wait duration for thread pools", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_avg( "tikv_yatp_pool_schedule_wait_duration", by_labels=["name", "priority"], ), legend_format="{{name}}-{{priority}}", additional_groupby=True, ), target( expr=expr_simple( "tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds", ), legend_format="sched-tokio-pool-mean-{{instance}}", ), target( expr=expr_simple( "tikv_tokio_pool_mean_first_poll_delay_seconds", label_selectors=['name="unified-read-pool"'], ), legend_format="unified-read-pool-tokio-mean-{{instance}}", ), ], thresholds=[GraphThreshold(value=1.0)], ), ] ) layout.row( [ graph_panel( title="Disk IO time per second", yaxes=yaxes(left_format=UNITS.NANO_SECONDS), lines=False, stack=True, targets=[ target( expr=expr_sum_rate( "tikv_storage_rocksdb_perf", label_selectors=['metric="block_read_time"'], by_labels=["req"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_coprocessor_rocksdb_perf", label_selectors=['metric="block_read_time"'], by_labels=["req"], ), legend_format="copr-{{req}}", additional_groupby=True, ), ], ), graph_panel( title="Disk IO bytes per second", yaxes=yaxes(left_format=UNITS.NANO_SECONDS), lines=False, stack=True, targets=[ target( expr=expr_sum_rate( "tikv_storage_rocksdb_perf", label_selectors=['metric="block_read_byte"'], by_labels=["req"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_coprocessor_rocksdb_perf", label_selectors=['metric="block_read_byte"'], by_labels=["req"], ), legend_format="copr-{{req}}", additional_groupby=True, ), ], ), ] ) return layout.row_panel def gRPC() -> RowPanel: layout = Layout(title="gRPC") layout.row( [ graph_panel( title="gRPC message count", description="The count of different kinds of gRPC message", yaxes=yaxes(left_format=UNITS.REQUESTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_grpc_msg_duration_seconds_count", label_selectors=['type!="kv_gc"'], by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_grpc_msg_duration_seconds_count", label_selectors=['type!="kv_gc"'], by_labels=["type", "priority"], ), hide=True, additional_groupby=True, ), ], ), graph_panel( title="gRPC message failed", description="The count of different kinds of gRPC message which is failed", yaxes=yaxes(left_format=UNITS.REQUESTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_grpc_msg_fail_total", label_selectors=['type!="kv_gc"'], by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title=r"gRPC message duration" + OPTIONAL_QUANTILE_INPUT, description=r"The " + OPTIONAL_QUANTILE_INPUT + " of execution time of gRPC message", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_grpc_msg_duration_seconds", label_selectors=['type!="kv_gc"'], by_labels=["type"], is_optional_quantile=True, ), legend_format="{{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_grpc_msg_duration_seconds", label_selectors=['type!="kv_gc"'], by_labels=["type", "priority"], is_optional_quantile=True, ), legend_format="{{type}}-{{priority}}-" + OPTIONAL_QUANTILE_INPUT, hide=True, additional_groupby=True, ), ], ), graph_panel( title="Average gRPC message duration", description="The average execution time of gRPC message", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_avg( "tikv_grpc_msg_duration_seconds", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_grpc_msg_duration_seconds", by_labels=["type", "priority"], ), legend_format="{{type}}-{{priority}}", hide=True, additional_groupby=True, ), ], ), graph_panel( title=r"gRPC batch commands wait duration", description=r"The 99.99% wait time of gRPC batch commands", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.9999, "tikv_grpc_batch_commands_wait_duration_seconds", is_optional_quantile=False, ), legend_format="P9999", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="gRPC batch size", description=r"The " + OPTIONAL_QUANTILE_INPUT + " of execution time of gRPC message", targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_server_grpc_req_batch_size", is_optional_quantile=True, ), legend_format=r"request-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_server_grpc_resp_batch_size", is_optional_quantile=True, ), legend_format=r"response-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_server_grpc_req_batch_size", by_labels=[], # override default by instance. ), legend_format="avg request", additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_server_grpc_resp_batch_size", by_labels=[], # override default by instance. ), legend_format="avg response", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_server_request_batch_size", is_optional_quantile=True, ), legend_format=r"kv get batch-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_server_request_batch_size", by_labels=[], # override default by instance. ), legend_format="avg kv batch", additional_groupby=True, ), ], ), graph_panel( title="raft message batch size", targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_server_raft_message_batch_size", is_optional_quantile=True, ), legend_format=OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_server_raft_message_batch_size", by_labels=[], # override default by instance. ), legend_format="avg", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="gRPC request sources QPS", description="The QPS of different sources of gRPC request", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_grpc_request_source_counter_vec", by_labels=["source"], ), additional_groupby=True, ), ], ), graph_panel( title="gRPC request sources duration", description="The duration of different sources of gRPC request", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), lines=False, stack=True, targets=[ target( expr=expr_sum_rate( "tikv_grpc_request_source_duration_vec", by_labels=["source"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="gRPC resource group QPS", description="The QPS of different resource groups of gRPC request", targets=[ target( expr=expr_sum_rate( "tikv_grpc_resource_group_total", by_labels=["name"] ), additional_groupby=True, ), ], ), stat_panel( title="gRPC Compression Status for Msg", targets=[ target( expr=expr_simple("tikv_grpc_compression_type_for_msg"), legend_format="{{ instance }}", ), ], mappings=[ StatValueMappings( StatValueMappingItem("none", "0", "green"), StatValueMappingItem("deflate", "1", "yellow"), StatValueMappingItem("gzip", "2", "red"), ), ], ), stat_panel( title="gRPC Compression Status for Raft Replication", targets=[ target( expr=expr_simple("tikv_grpc_compression_type_for_raft"), legend_format="{{ instance }}", ), ], mappings=[ StatValueMappings( StatValueMappingItem("none", "0", "green"), StatValueMappingItem("deflate", "1", "yellow"), StatValueMappingItem("gzip", "2", "red"), ), ], ), ] ) return layout.row_panel def ThreadCPU() -> RowPanel: layout = Layout(title="Thread CPU") layout.row( thread_cpu_panel_pair( title="Raft store CPU", description="The CPU utilization of raftstore thread", label_selector='name=~"(raftstore|rs|rfstore)_.*"', ) ) layout.row( thread_cpu_panel_pair( title="Async apply CPU", description="The CPU utilization of async apply", label_selector='name=~"apply_[0-9]+|apply_follower_[0-9]+"', by_labels=["instance", "name"], ) ) layout.row( thread_cpu_panel_pair( title="Raft async writer CPU", description="The CPU utilization of raft log writer thread", label_selector='name=~"raft_io.*"', thresholds=[GraphThreshold(value=0.8)], ) ) layout.row( thread_cpu_panel_pair( title="Scheduler pool CPU", description="The CPU utilization of the scheduler pools", label_selector='name=~"sched_tokio_pool.*|sched_yatp_pool.*|sched_background.*"', ) ) layout.row( thread_cpu_panel_pair( title="gRPC poll CPU", description="The CPU utilization of gRPC", label_selector='name=~"grpc.*"', ) ) layout.row( thread_cpu_panel_pair( title="KvEngine worker CPU", label_selector='name=~"txn_chunk_worker.*"', ) ) layout.row( thread_cpu_panel_pair( title="RfEngine worker CPU", label_selector='name=~"sub_compact_wkr.*|compact_wal_worker.*"', ) ) layout.row( thread_cpu_panel_pair( title="Unified read pool CPU", description="The CPU utilization of the unified read pool", label_selector='name=~"unified_read_po.*"', thresholds=[GraphThreshold(value=7.2)], ) ) layout.row( thread_cpu_panel_pair( title="Background worker CPU", label_selector='name=~"background.*"', ) ) layout.row( thread_cpu_panel_pair( title="GC worker CPU", label_selector='name=~"gc_worker.*"', ) ) layout.row( thread_cpu_panel_pair( title="Import CPU", label_selector='name=~"sst_.*"', ) ) layout.row( thread_cpu_panel_pair( title="Backup CPU", label_selector='name=~"(backup_worker|bkwkr|backup_endpoint).*"', ) ) layout.row( thread_cpu_panel_pair( title="CDC worker CPU", label_selector='name=~"(cdcwkr.*|tso|cdc_.*)"', by_labels=["instance", "name"], ) ) layout.row( thread_cpu_panel_pair( title="TSO worker CPU", description="The CPU utilization of raftstore thread", label_selector='name=~"tso_worker"', ) ) layout.row( thread_cpu_panel_pair( title="IME CPU", description="The CPU utilization of IME threads", label_selector='name=~"ime.*"', by_labels=["instance", "name"], ) ) layout.row( thread_cpu_panel_pair( title="IA CPU", description="The CPU utilization of IA threads", label_selector='name=~"s3fifo.*"', ) ) layout.row( thread_cpu_panel_pair( title="DFS worker CPU", description="The CPU utilization of DFS worker threads", label_selector='name=~"(builtin_dfs.*|memory_fs.*|s3_client.*)"', ) ) layout.row( thread_cpu_panel_pair( title="Cloud worker CPU", description="The CPU utilization of Cloud worker threads", label_selector='name=~"worker_server.*"', ) ) layout.row( [ graph_panel( title="KvEngine worker CPU", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( "tikv_thread_cpu_seconds_total", label_selectors=['name=~"txn_chunk_worker.*"'], ), ), ], ), graph_panel( title="Busy Threads (>80%)", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_topk( 20, "%s" % expr_sum_rate( "tikv_thread_cpu_seconds_total", label_selectors=['name!~"rocksdb.*"'], by_labels=["instance", "name"], ).extra(extra_expr="> 0.8"), ), legend_format="{{name}}-{{instance}}", ), ], ), ] ) return layout.row_panel def WorkerThreadCPU() -> RowPanel: layout = Layout(title="Thread CPU") layout.row( thread_cpu_panel_pair( title="Worker Threads CPU", description="The CPU utilization of TiKV and coprocessor worker threads", label_selector='name=~".+"', by_labels=["container", "instance", "name"], metric_namespace="tikv_worker", ) ) return layout.row_panel def TTL() -> RowPanel: layout = Layout(title="TTL") layout.row( [ graph_panel( title="TTL expire count", targets=[ target( expr=expr_sum_rate( "tikv_ttl_expire_kv_count_total", ), ), ], ), graph_panel( title="TTL expire size", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_ttl_expire_kv_size_total", ), ), ], ), ] ) layout.row( [ graph_panel( title="TTL check progress", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_sum_rate( "tikv_ttl_checker_processed_regions", ), "/", expr_sum_rate( "tikv_raftstore_region_count", label_selectors=['type="region"'], ), ), legend_format="{{instance}}", ), ], ), graph_panel( title="TTL checker actions", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_ttl_checker_actions", by_labels=["type"] ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="TTL checker compact duration", description="The time consumed when executing GC tasks", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_ttl_checker_compact_duration", ), stat_panel( title="TTL checker poll interval", format=UNITS.MILLI_SECONDS, targets=[ target( expr=expr_max( "tikv_ttl_checker_poll_interval", label_selectors=['type="tikv_gc_run_interval"'], by_labels=[], # override default by instance. ), ), ], ), ] ) return layout.row_panel def PD() -> RowPanel: layout = Layout(title="PD") layout.row( [ graph_panel( title="PD requests", description="The count of requests that TiKV sends to PD", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_pd_request_duration_seconds_count", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="PD request duration (average)", description="The time consumed by requests that TiKV sends to PD", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_pd_request_duration_seconds", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="PD heartbeats", description="The total number of PD heartbeat messages", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_pd_heartbeat_message_total", by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum( "tikv_pd_pending_heartbeat_total", ), legend_format="{{instance}}-pending", ), ], ), graph_panel( title="PD validate peers", description="The total number of peers validated by the PD worker", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_pd_validate_peer_total", by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="PD reconnection", description="The count of reconnection between TiKV and PD", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_delta( "tikv_pd_reconnect_total", range_selector="$__rate_interval", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="PD forward status", description="The forward status of PD client", targets=[ target( expr=expr_simple( "tikv_pd_request_forwarded", ), legend_format="{{instance}}-{{host}}", ), ], ), ] ) layout.row( [ graph_panel( title="Pending TSO Requests", description="The number of TSO requests waiting in the queue.", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum( "tikv_pd_pending_tso_request_total", ), additional_groupby=True, ), ], ), ] ) return layout.row_panel def IOBreakdown() -> RowPanel: layout = Layout(title="IO Breakdown") layout.row( [ graph_panel( title="Write IO bytes", description="The throughput of disk write per IO type", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_io_bytes", label_selectors=['op="write"'], by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_io_bytes", label_selectors=['op="write"'], by_labels=[], # override default by instance. ), legend_format="total", additional_groupby=True, ), ], ), graph_panel( title="Read IO bytes", description="The throughput of disk read per IO type", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_io_bytes", label_selectors=['op="read"'], by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_io_bytes", label_selectors=['op="read"'], by_labels=[], # override default by instance. ), legend_format="total", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="IO threshold", description="The threshold of disk IOs per priority", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_avg( "tikv_rate_limiter_max_bytes_per_sec", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Rate Limiter Request Wait Duration", description="IO rate limiter request wait duration.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_rate_limiter_request_wait_duration_seconds", by_labels=["type"], is_optional_quantile=True, ), legend_format=r"{{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_rate_limiter_request_wait_duration_seconds", by_labels=[], # override default by instance. ), legend_format="avg", additional_groupby=True, ), ], ), ] ) return layout.row_panel def RaftWaterfall() -> RowPanel: layout = Layout(title="Raft Waterfall") layout.row( [ graph_panel_histogram_quantiles( title="Storage async write duration", description="The time consumed by processing asynchronous write requests", yaxes=yaxes(left_format=UNITS.SECONDS, right_format=UNITS.NONE_FORMAT), metric="tikv_storage_engine_async_request_duration_seconds", label_selectors=['type="write"'], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Store duration", description="The store time duration of each request", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_duration_secs", ), graph_panel_histogram_quantiles( title="Apply duration", description="The apply time duration of each request", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_apply_duration_secs", ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Store propose wait duration", description="Time from request scheduling to when it is handled by Raftstore", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_request_wait_time_duration_secs", ), graph_panel_histogram_quantiles( title="Store batch wait duration", description="Time from request scheduling to when a batch of requests is formed and prepared to be proposed to Raft", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_batch_wait_duration_seconds", ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Store send to write queue duration", description="Time from request scheduling to just before it is sent to the store writer thread", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_send_to_queue_duration_seconds", ), graph_panel_histogram_quantiles( title="Store send proposal duration", description="Time from request scheduling to just before it is sent to followers", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_send_proposal_duration_seconds", ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Store before write duration", description="Time from request scheduling to just before it is written to Raft Engine", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_before_write_duration_seconds", ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Store write end duration", description="Time from request scheduling to when it is written to Raft Engine", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_write_end_duration_seconds", ), graph_panel_histogram_quantiles( title="Store persist duration", description="Time from request scheduling to when its associated ready is persisted on the leader", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_persist_duration_seconds", ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Store commit but not persist duration", description="Time from request scheduling to when it is committed; at the time of commit, it has not been persisted on the leader", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds", ), graph_panel_histogram_quantiles( title="Store commit and persist duration", description="Time from request scheduling to when it is committed; at the time of commit, it has been persisted on the leader", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_raftstore_store_wf_commit_log_duration_seconds", ), ] ) return layout.row_panel def RaftIO() -> RowPanel: layout = Layout(title="Raft IO") layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Process ready duration", heatmap_description="The time taken by Raftstore to complete processing a poll round, which includes a batch of region peers", graph_title="99% Process ready duration per server", graph_description="The time taken by Raftstore to complete processing a poll round, which includes a batch of region peers", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_raft_process_duration_secs", label_selectors=['type="ready"'], ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Store write send duration", heatmap_description="The duration taken by sending raft logs to channels of followers.", graph_title="99% Store write send duration per server", graph_description="The duration taken by sending raft logs to channels of followers.", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_store_write_send_duration_seconds", ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Append log duration", heatmap_description="The time consumed when Raft appends log", graph_title="99% Append log duration per server", graph_description="The time consumed when Raft appends log on each TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_store_write_raftdb_duration_seconds", ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Commit log duration", heatmap_description="The time consumed when Raft commits log", graph_title="99% Commit log duration per server", graph_description="The time consumed when Raft commits log on each TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_commit_log_duration_seconds", ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Apply log duration", heatmap_description="The time consumed when Raft applies log", graph_title="99% Apply log duration per server", graph_description="The time consumed for Raft to apply logs per TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_apply_log_duration_seconds", ) ) return layout.row_panel def RaftPropose() -> RowPanel: layout = Layout(title="Raft Propose") layout.row( [ graph_panel( title="Raft proposals per ready", description="The proposal count of a Regions in a tick", targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_apply_proposal", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-" + OPTIONAL_QUANTILE_INPUT, ), ], ), graph_panel( title="Raft read/write proposals", description="The number of proposals per type", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_proposal_total", label_selectors=['type=~"local_read|normal|read_index"'], by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Raft read proposals per server", description="The number of read proposals which are made by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_proposal_total", label_selectors=['type=~"local_read|read_index"'], ), ), ], ), graph_panel( title="Raft write proposals per server", description="The number of write proposals which are made by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_proposal_total", label_selectors=['type=~"normal"'], ), ), ], ), ] ) layout.row( [ graph_panel( title="Raft write apply", description="The number of write apply", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_write_cmd_total", by_labels=["type"], ), ), ], ), graph_panel( title="Raft change_set apply", description="The number of change_set apply", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kvengine_apply_change_set_total", by_labels=["type"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Raft change set proposals per server", description="The number of change set proposals which are made by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_proposal_total", label_selectors=['type=~"change_set"'], ), ), ], ), graph_panel( title="Raft change set apply per server", description="The number of change set apply which are made by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_write_cmd_total", label_selectors=['type=~"change_set|shard_ingest"'], ), ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Propose wait duration", heatmap_description="The wait time of each proposal", graph_title="99% Propose wait duration per server", graph_description="The wait time of each proposal in each TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_request_wait_time_duration_secs", ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Store write wait duration", heatmap_description="The wait time of each store write task", graph_title="99% Store write wait duration per server", graph_description="The wait time of each store write task in each TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_store_write_task_wait_duration_secs", ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Apply wait duration", heatmap_description="The wait time of each apply task", graph_title="99% Apply wait duration per server", graph_description="The wait time of each apply task in each TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_raftstore_apply_wait_time_duration_secs", ) ) layout.row( [ heatmap_panel( title="Store write handle msg duration", description="The handle duration of each store write task msg", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_raftstore_store_write_handle_msg_duration_secs_bucket", ), heatmap_panel( title="Store write trigger size", description="The distribution of write trigger size", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), metric="tikv_raftstore_store_write_trigger_wb_bytes_bucket", ), ] ) layout.row( [ graph_panel( title="Raft propose speed", description="The rate at which peers propose logs", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_propose_log_size_sum", ), ), ], ), graph_panel( title="Perf Context duration", description="The rate at which peers propose logs", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_store_perf_context_time_duration_secs", by_labels=["type"], is_optional_quantile=True, ), legend_format="store-{{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_apply_perf_context_time_duration_secs", by_labels=["type"], is_optional_quantile=True, ), legend_format="apply-{{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) return layout.row_panel def RaftProcess() -> RowPanel: layout = Layout(title="Raft Process") layout.row( [ graph_panel( title="Ready handled", description="The count of different ready type of Raft", targets=[ target( expr=expr_sum_rate( "tikv_raftstore_raft_ready_handled_total", by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_raftstore_raft_process_duration_secs_count", label_selectors=['type="ready"'], by_labels=[], # overwrite default by instance. ), legend_format="count", additional_groupby=True, ), ], ), graph_panel( title="Max duration of raft store events", description="The max time consumed by raftstore events", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.999999, "tikv_raftstore_event_duration", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.999999, "tikv_broadcast_normal_duration_seconds", ), legend_format="broadcast_normal", additional_groupby=True, ), ], ), ] ) layout.row( [ heatmap_panel( title="rfstore batch message count", description="Number of messages polled in one batch by the main peer worker", metric="rfstore_receive_msgs_count_bucket", ), heatmap_panel( title="rfstore batch message size", description="Size of messages polled in one batch by the main peer worker", metric="rfstore_receive_msgs_size_bytes_bucket", y_axis=yaxis(format=UNITS.SHORT, decimals=1), ), ] ) layout.row( [ graph_panel( title="rfstore main worker time breakdown", description="Time spent in different steps of the rfstore main worker loop", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "rfstore_store_msg_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-handle-store-msg-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "rfstore_receive_msgs_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-receive-msgs-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "rfstore_sync_aux_worker_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-sync-aux-workers-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "rfstore_send_aux_task_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-send-aux-tasks-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "rfstore_process_msgs_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-process-msgs-" + OPTIONAL_QUANTILE_INPUT, ), ], ), heatmap_panel( title="Replica read lock checking duration", description="Replica read lock checking duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_replica_read_lock_check_duration_seconds_bucket", ), ] ) layout.row( [ heatmap_panel( title="Peer msg length distribution", description="The length of peer msgs for each round handling", metric="tikv_raftstore_peer_msg_len_bucket", ), heatmap_panel( title="Apply msg length distribution", description="The length of apply msgs for each round handling", metric="tikv_raftstore_apply_msg_len_bucket", ), ] ) layout.row( [ graph_panel( title="Enable apply unpersisted log region count", description="The number of regions that enable apply unpersisted raft log", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_simple( "tikv_raft_enable_unpersisted_apply_regions", ), legend_format="{{instance}}", ), ], ), graph_panel( title="Apply ahead of persistence raft log count", description="The number of raft logs between apply and persisted index", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_raft_apply_ahead_of_persist", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 1, "tikv_raft_apply_ahead_of_persist", by_labels=["instance"], ), legend_format="{{instance}}-max", ), ], ), ] ) layout.row( [ graph_panel( title="Apply Poll Handle Duration", description="The duration of each apply future task running in one round", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_rfstore_apply_running_duration_secs", by_labels=["instance"], ), legend_format="{{instance}}-P99", ) ], ), graph_panel( title="Apply Poll Handle Batch Count", description="The number of apply batch a apply future task handles", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_rfstore_apply_handle_msgs_batch_number", by_labels=["instance"], ), legend_format="{{instance}}-P99", ), target( expr=expr_histogram_avg( "tikv_rfstore_apply_handle_msgs_batch_number", by_labels=["instance"], ), legend_format="{{instance}}-avg", ), ], ), ] ) return layout.row_panel def RaftMessage() -> RowPanel: layout = Layout(title="Raft Message") layout.row( [ graph_panel( title="Sent messages per server", description="The number of Raft messages sent by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_raft_sent_message_total", ), ), ], ), graph_panel( title="Flush messages per server", description="The number of Raft messages flushed by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_server_raft_message_flush_total", by_labels=["instance", "reason"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Receive messages per server", description="The number of Raft messages received by each TiKV instance", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_server_raft_message_recv_total", ), ), ], ), graph_panel( title="Messages", description="The number of different types of Raft messages that are sent", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_raft_sent_message_total", label_selectors=['status="accept"'], by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Vote", description="The total number of vote messages that are sent in Raft", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_raft_sent_message_total", label_selectors=['type="vote"'], ), ), ], ), graph_panel( title="Raft dropped messages", description="The number of dropped Raft messages per type", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_raft_dropped_message_total", by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_raftstore_raft_sent_message_total", label_selectors=['status="drop"'], by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_rfstore_router_raft_message_dropped_total", by_labels=["type"], ), legend_format="router_{{type}}", additional_groupby=True, ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Raft Message Send Wait duration", heatmap_description="The time consumed waiting to send Raft Messages", graph_title="99% Raft Message Send Wait Duration", graph_description="The time consumed waiting to send Raft Messages per TiKV instance", graph_by_labels=["instance"], yaxis_format=UNITS.SECONDS, metric="tikv_server_raft_message_duration_seconds", label_selectors=['type="send_wait"'], ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Raft Message Receive Delay duration", heatmap_description="The time consumed to transmit Raft Messages over the network, reported by the receiver", graph_title="99% Raft Message Receive Delay Duration", graph_description="The time consumed to transmit Raft Messages over the network per TiKV instance, reported by the receiver", graph_by_labels=["instance"], yaxis_format=UNITS.SECONDS, metric="tikv_server_raft_message_duration_seconds", label_selectors=['type="receive_delay"'], ) ) return layout.row_panel def RaftAdmin() -> RowPanel: layout = Layout(title="Raft Admin") layout.row( [ graph_panel( title="Admin proposals", description="The number of admin proposals", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_proposal_total", label_selectors=['type=~"conf_change|transfer_leader"'], by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Admin apply", description="The number of the processed apply command", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_admin_cmd_total", label_selectors=['type!="compact"'], by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Check split", description="The number of raftstore split checks", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_raftstore_check_split_total", label_selectors=['type!="ignore"'], by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="99.99% Check split duration", description="The time consumed when running split check in .9999", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.9999, "tikv_raftstore_check_split_duration_seconds", by_labels=["instance"], ), legend_format="{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="Load base split event", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_delta( "tikv_load_base_split_event", range_selector="1m", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Load base split duration", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.80, "tikv_load_base_split_duration_seconds", by_labels=["instance"], ), legend_format="80%-{{instance}}", ), target( expr=expr_histogram_quantile( 0.99, "tikv_load_base_split_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_avg( "tikv_load_base_split_duration_seconds", by_labels=["instance"], ), legend_format="avg-{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="Observed region CPU for load-base split", description=( "Per-region CPU observed before load-fit filtering when " "load-base split evaluates regions. Unit follows other CPU " "panels: 100% means one CPU core. This is pre-load-fit " "decision input; use the tail distribution to tune split " "thresholds. Use additional_groupby=instance only when " "drilling into a specific TiKV; keep it as none for large " "clusters." ), yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_histogram_quantile( 0.9999, "tikv_load_base_split_region_load", label_selectors=['type="cpu_millicores"'], ), "/", "1000", ), legend_format="99.99%", additional_groupby=True, ), target( expr=expr_operator( expr_histogram_quantile( 0.99, "tikv_load_base_split_region_load", label_selectors=['type="cpu_millicores"'], ), "/", "1000", ), legend_format="99%", additional_groupby=True, ), target( expr=expr_operator( expr_histogram_avg( "tikv_load_base_split_region_load", label_selectors=['type="cpu_millicores"'], by_labels=[], ), "/", "1000", ), legend_format="avg", additional_groupby=True, ), ], ), graph_panel_histogram_quantiles( title="Observed region QPS for load-base split", description=( "Per-region QPS observed before load-fit filtering when " "load-base split evaluates regions. This is pre-load-fit " "decision input; use the tail distribution to tune split " "thresholds. Use additional_groupby=instance only when " "drilling into a specific TiKV; keep it as none for large " "clusters." ), yaxes=yaxes(left_format=UNITS.REQUESTS_PER_SEC), metric="tikv_load_base_split_region_load", label_selectors=['type="qps"'], hide_count=True, additional_groupby=True, ), graph_panel_histogram_quantiles( title="Observed region read bytes for load-base split", description=( "Per-region read bytes observed before load-fit filtering " "when load-base split evaluates regions. Unit: KiB. This is " "pre-load-fit decision input; use the tail distribution to " "tune split thresholds. Use additional_groupby=instance only " "when drilling into a specific TiKV; keep it as none for " "large clusters." ), yaxes=yaxes(left_format=UNITS.KIBI_BYTES), metric="tikv_load_base_split_region_load", label_selectors=['type="bytes_kib"'], hide_count=True, additional_groupby=True, ), ] ) layout.row( [ graph_panel( title="Peer in Flashback State", targets=[ target( expr=expr_sum( "tikv_raftstore_peer_in_flashback_state", ), ), ], ), ] ) return layout.row_panel def RaftLog() -> RowPanel: layout = Layout(title="Raft Log") layout.row( [ graph_panel( title="Raft log fetch", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "rfstore_raft_entry_fetches_total", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Raft log async fetch task duration", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=10), targets=[ target( expr=expr_histogram_quantile( 0.9999, "rfstore_raft_entry_fetches_task_duration_seconds", ), legend_format="99.99%", additional_groupby=True, ), target( expr=expr_histogram_avg( "rfstore_raft_entry_fetches_task_duration_seconds", by_labels=["instance"], ), legend_format="avg-{{instance}}", ), target( expr=expr_sum( "tikv_worker_pending_task_total", label_selectors=['name=~"async-read-worker"'], ), legend_format="pending-task-{{instance}}", additional_groupby=True, ), ], series_overrides=[ series_override( alias="/pending-task/", yaxis=2, transform_negative_y=True, ), ], ), ] ) layout.row( [ graph_panel( title="Raft log lag", targets=[ target( expr=expr_sum_rate( "tikv_raftstore_log_lag_sum", ), ), ], ), ] ) return layout.row_panel def LocalReader() -> RowPanel: layout = Layout(title="Local Reader") layout.row( [ graph_panel( title="Local reader requests", targets=[ target( expr=expr_sum_rate( "tikv_raftstore_local_read_reject_total", by_labels=["instance", "reason"], ), legend_format="{{instance}}-reject-by-{{reason}}", ), target( expr=expr_sum_rate( "tikv_raftstore_local_read_executed_requests", ), legend_format="{{instance}}-total", ), target( expr=expr_sum_rate( "tikv_raftstore_local_read_executed_stale_read_requests", ), legend_format="{{instance}}-stale-read", ), ], series_overrides=[ series_override( alias="/.*-total/", yaxis=2, ), ], ), ] ) return layout.row_panel def UnifiedReadPool() -> RowPanel: layout = Layout(title="Unified Read Pool") yatp_keyspace_label_selectors = [ 'name=~"unified-read.*"', 'keyspace_id=~"$keyspace_id"', ] yatp_keyspace_by_labels = ["keyspace_id"] def keyspace_task_time_per_slice_expr(metric: str): return expr_operator( expr_operator( expr_sum_rate( metric, label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, ), "/", expr_sum_rate( "tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, ), ), "*", "1000", ) # Row 1: Wait/Schedule Duration + Running Tasks layout.row( [ graph_panel( title="Schedule Wait Duration", description="Time from task spawn to first poll", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_yatp_pool_schedule_wait_duration", label_selectors=['name=~"unified-read.*"'], ), legend_format="mean-yatp-{{instance}}", ), target( expr=expr_simple( "tikv_tokio_pool_mean_first_poll_delay_seconds", label_selectors=['name="unified-read-pool"'], ), legend_format="mean-tokio-{{instance}}", ), ], ), graph_panel( title="Running Tasks", description="Number of concurrently running tasks", targets=[ target( expr=expr_sum_aggr_over_time( "tikv_unified_read_pool_running_tasks", "avg", "1m", by_labels=["priority"], ), additional_groupby=True, ), ], ), ] ) # Row 2: Poll Duration + Scheduled Duration layout.row( [ graph_panel( title="Task Poll Duration", description="Time tasks spend executing (being polled)", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_yatp_task_poll_duration", label_selectors=['name=~"unified-read.*"'], ), legend_format="mean-yatp-{{instance}}", ), target( expr=expr_simple( "tikv_tokio_pool_mean_poll_duration_seconds", label_selectors=['name="unified-read-pool"'], ), legend_format="mean-tokio-{{instance}}", ), ], ), graph_panel( title="Task Scheduled Duration", description="Time tasks wait to be polled after being woken", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_simple( "tikv_tokio_pool_mean_scheduled_duration_seconds", label_selectors=['name="unified-read-pool"'], ), legend_format="mean-tokio-{{instance}}", ), ], ), ] ) # Row 3: Idle Duration + First Poll Count layout.row( [ graph_panel( title="Task Idle Duration", description="Time tasks spend idle waiting on external events (I/O)", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_simple( "tikv_tokio_pool_mean_idle_duration_seconds", label_selectors=['name="unified-read-pool"'], ), legend_format="mean-tokio-{{instance}}", ), ], ), graph_panel( title="First Poll Count", description="Rate of tasks polled for first time", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_tokio_pool_first_poll_total", label_selectors=['name="unified-read-pool"'], ), legend_format="tokio-{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="Keyspace Task CPU / Wall Time Rate", description="CPU and wall time consumed by YATP read-pool task polls for each keyspace. 1.0 means one second per second.", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, ), legend_format="cpu-KSID-{{keyspace_id}}", ), target( expr=expr_sum_rate( "tikv_yatp_pool_keyspace_task_wall_seconds_total", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, ), legend_format="wall-KSID-{{keyspace_id}}", ), ], ), graph_panel_histogram_quantiles( title="Keyspace Fair Queue Slice Wait Duration", description="Duration from pushing a task slice into the YATP keyspace fair queue to popping it for each keyspace.", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, hide_p9999=True, hide_count=True, additional_groupby=False, legend_label_prefix="KSID-", ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Keyspace Fair Queue Length", description="Per-keyspace queued task slices in the YATP keyspace fair queue before a task slice is pushed.", yaxes=yaxes(left_format=UNITS.SHORT), metric="tikv_yatp_pool_keyspace_fair_queue_length", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, hide_p9999=True, hide_count=True, additional_groupby=False, legend_label_prefix="KSID-", ), heatmap_panel( title="Keyspace Fair Queue Task Slice Waits", description="Number of successful task-slice pops after a task slice is pushed and before it is popped from the YATP keyspace fair queue. Multiple selected keyspaces are aggregated in this heatmap.", metric="tikv_yatp_pool_keyspace_fair_queue_task_slice_waits_bucket", label_selectors=yatp_keyspace_label_selectors, y_axis=yaxis(format=UNITS.SHORT, decimals=0), ), ] ) layout.row( [ graph_panel( title="Keyspace Avg Task CPU / Wall Time Per Slice", description="Average CPU and wall time consumed by each popped YATP read-pool task slice for each keyspace.", yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=keyspace_task_time_per_slice_expr( "tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum" ), legend_format="cpu-KSID-{{keyspace_id}}", ), target( expr=keyspace_task_time_per_slice_expr( "tikv_yatp_pool_keyspace_task_wall_seconds_total" ), legend_format="wall-KSID-{{keyspace_id}}", ), ], ), graph_panel( title="Active Fair Keyspace Groups", description="Current number of keyspace groups in the YATP keyspace fair queue.", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_max( "tikv_yatp_pool_keyspace_fair_queue_group_count", label_selectors=['name=~"unified-read.*"'], by_labels=["instance"], ), legend_format="{{ instance }}", ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Keyspace Task Slice CPU Time P99", description="P99 CPU time spent in one YATP read-pool task slice for each keyspace.", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, hide_p9999=True, hide_avg=True, hide_count=True, additional_groupby=False, legend_label_prefix="KSID-", ), graph_panel_histogram_quantiles( title="Keyspace Task Slice Schedule Interval P99", description="P99 interval from the later of the previous pop and current enqueue to the current task-slice pop for each keyspace.", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds", label_selectors=yatp_keyspace_label_selectors, by_labels=yatp_keyspace_by_labels, hide_p9999=True, hide_avg=True, hide_count=True, additional_groupby=False, legend_label_prefix="KSID-", ), ] ) # Row 4: EXTRA_CF scan iterations layout.row( [ graph_panel_histogram_quantiles( title="EXTRA CF Scan Examined Entries", description="Histogram of entries examined during EXTRA_CF conflict checks", yaxes=yaxes( left_format=UNITS.NONE_FORMAT, right_format=UNITS.OPS_PER_SEC ), metric="tikv_storage_extra_cf_scan_iterations", hide_p9999=True, ), ] ) return layout.row_panel def Yatp() -> RowPanel: layout = Layout(title="YATP") layout.row( [ graph_panel( title="Time Used by Level", description="The execution time used by each level in YATP thread pools per second. Level 0 refers to small queries.", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_sum_rate( "tikv_multilevel_level_elapsed", by_labels=["name", "level"], ), additional_groupby=True, ), ], ), graph_panel( title="Level 0 Chance", description="The chance that level 0 (small) tasks are scheduled in YATP thread pools.", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_simple( "tikv_multilevel_level0_chance", ), legend_format="{{name}}-{{instance}}", ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Task Wait Duration", description="Time a completed task spent waiting in a YATP thread pool.", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="tikv_yatp_task_wait_duration", by_labels=["name"], hide_count=True, ), graph_panel_histogram_quantiles( title="Task Execute Duration", description="Total execution time across all polls of a completed YATP task.", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="tikv_yatp_task_exec_duration", by_labels=["name"], hide_count=True, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Duration of One Time Slice", description="Execution time of a single YATP task poll at each level.", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="tikv_yatp_task_poll_duration", by_labels=["name", "level"], hide_count=True, ), graph_panel_histogram_quantiles( title="Task Schedule Times", description="Number of times a completed YATP task was polled before completion.", yaxes=yaxes(left_format=UNITS.NONE_FORMAT, log_base=2), metric="tikv_yatp_task_execute_times", by_labels=["name"], hide_count=True, ), ] ) return layout.row_panel def Storage() -> RowPanel: layout = Layout(title="Storage") layout.row( [ graph_panel( title="Storage command total", description="The total count of different kinds of commands received", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC, log_base=10), targets=[ target( expr=expr_sum_rate( "tikv_storage_command_total", by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Storage async request error", description="The total number of engine asynchronous request errors", targets=[ target( expr=expr_sum_rate( "tikv_storage_engine_async_request_total", label_selectors=['status!~"all|success"'], by_labels=["status"], ), additional_groupby=True, ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Storage async write duration", heatmap_description="The time consumed by processing asynchronous write requests", graph_title="Storage async write duration", graph_description="The storage async write duration", yaxis_format=UNITS.SECONDS, metric="tikv_storage_engine_async_request_duration_seconds", label_selectors=['type="write"'], ), ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Storage async snapshot duration", heatmap_description="The time consumed by processing asynchronous snapshot requests", graph_title="Storage async snapshot duration", graph_description="The storage async snapshot duration", yaxis_format=UNITS.SECONDS, metric="tikv_storage_engine_async_request_duration_seconds", label_selectors=['type="snapshot"'], ), ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Storage async snapshot duration (pure local read)", heatmap_description="The storage async snapshot duration without the involving of raftstore", graph_title="Storage async snapshot duration (pure local read)", graph_description="The storage async snapshot duration without the involving of raftstore", yaxis_format=UNITS.SECONDS, metric="tikv_storage_engine_async_request_duration_seconds", label_selectors=['type="snapshot_local_read"'], ), ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Read index propose wait duration", heatmap_description="Read index propose wait duration associated with async snapshot", graph_title="Read index propose wait duration", graph_description="Read index propose wait duration associated with async snapshot", yaxis_format=UNITS.SECONDS, metric="tikv_storage_engine_async_request_duration_seconds", label_selectors=['type="snapshot_read_index_propose_wait"'], ), ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Read index confirm duration", heatmap_description="Read index confirm duration associated with async snapshot", graph_title="Read index confirm duration", graph_description="Read index confirm duration associated with async snapshot", yaxis_format=UNITS.SECONDS, metric="tikv_storage_engine_async_request_duration_seconds", label_selectors=['type="snapshot_read_index_confirm"'], ), ) layout.row( [ graph_panel( title="Concurrency manager max-ts", description="The max_ts in the concurrency manager", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr="tikv_concurrency_manager_max_ts_limit", legend_format="max_ts_limit-{{instance}}", ), target( expr="tikv_concurrency_manager_max_ts", legend_format="max_ts-{{instance}}", ), ], ) ] ) return layout.row_panel def FlowControl() -> RowPanel: layout = Layout(title="Flow Control") layout.row( [ graph_panel( title="Throttling counter", description="Counter when throttle happens per limiter", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_limiter_throttle_counter", by_labels=["limiter"], ), additional_groupby=True, ), ], ), graph_panel( title="Throttling algorithm actions", description="Counter when different throttling actions happen per limiter", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_limiter_throttle_action_counter", by_labels=["limiter", "type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Scheduler flow", description="", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_scheduler_write_flow", ), legend_format="write-{{instance}}", ), target( expr=expr_sum( "tikv_scheduler_throttle_flow", ).extra(" != 0"), legend_format="throttle-{{instance}}", ), ], ), heatmap_panel( title="Throttle duration", metric="tikv_scheduler_throttle_duration_seconds_bucket", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), ), ] ) return layout.row_panel def SchedulerCommands() -> RowPanel: layout = Layout(title="Scheduler", repeat="command") layout.row( [ graph_panel( title="Scheduler stage total", description="The total number of commands on each stage in commit command", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_too_busy_total", label_selectors=['type="$command"'], ), legend_format="busy-{{instance}}", ), target( expr=expr_sum_rate( "tikv_scheduler_stage_total", label_selectors=['type="$command"'], by_labels=["stage"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Scheduler command duration", description="The time consumed when executing commit command", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_scheduler_command_duration_seconds", label_selectors=['type="$command"'], hide_count=True, ), graph_panel_histogram_quantiles( title="Scheduler latch wait duration", description="The time which is caused by latch wait in commit command", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_scheduler_latch_wait_duration_seconds", label_selectors=['type="$command"'], hide_count=True, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Scheduler keys read", description="The count of keys read by a commit command", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), metric="tikv_scheduler_kv_command_key_read", label_selectors=['type="$command"'], hide_count=True, ), graph_panel_histogram_quantiles( title="Scheduler keys written", description="The count of keys written by a commit command", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), metric="tikv_scheduler_kv_command_key_write", label_selectors=['type="$command"'], hide_count=True, ), ] ) layout.row( [ graph_panel( title="Scheduler scan details", description="The keys scan details of each CF when executing commit command", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_kv_scan_details", label_selectors=['req="$command"'], by_labels=["tag"], ), additional_groupby=True, ), ], ), graph_panel( title="Scheduler scan details [lock]", description="The keys scan details of lock CF when executing commit command", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_kv_scan_details", label_selectors=['req="$command", cf="lock"'], by_labels=["tag"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Scheduler scan details [write]", description="The keys scan details of write CF when executing commit command", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_kv_scan_details", label_selectors=['req="$command", cf="write"'], by_labels=["tag"], ), additional_groupby=True, ), ], ), graph_panel( title="Scheduler scan details [default]", description="The keys scan details of default CF when executing commit command", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_kv_scan_details", label_selectors=['req="$command", cf="default"'], by_labels=["tag"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Scheduler command read duration", description="The time consumed on reading when executing commit command", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_scheduler_processing_read_duration_seconds", label_selectors=['type="$command"'], hide_count=True, ), heatmap_panel( title="Check memory locks duration", description="The time consumed on checking memory locks", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_storage_check_mem_lock_duration_seconds_bucket", label_selectors=['type="$command"'], ), ] ) return layout.row_panel def Scheduler() -> RowPanel: layout = Layout(title="Scheduler") layout.row( [ graph_panel( title="Scheduler stage total", description="The total number of commands on each stage", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_too_busy_total", by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_scheduler_stage_total", by_labels=["stage"], ), additional_groupby=True, ), ], ), graph_panel( title="Scheduler priority commands", description="The count of different priority commands", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_commands_pri_total", by_labels=["priority"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Scheduler pending commands", description="The count of pending commands per TiKV instance", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "tikv_scheduler_context_total", ), ), ], ), graph_panel( title="Scheduler running tasks", description="Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "tikv_scheduler_pool_running_tasks", ), ), ], ), ] ) layout.row( [ graph_panel( title="Scheduler writing bytes", description="The total writing bytes of commands on each stage", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_scheduler_writing_bytes", ), ), ], ), graph_panel( title="Scheduler memory quota", description="The number of bytes used by scheduler", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_scheduler_memory_quota_size", label_selectors=['type="in_use"'], ), ), target( expr=expr_sum( "tikv_scheduler_memory_quota_size", label_selectors=['type="capacity"'], ), hide=True, ), ], ), ] ) layout.row( [ graph_panel( title="Scheduler Pool Wait Duration", description="Time tasks wait to be scheduled (YATP or Tokio backend)", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_yatp_pool_schedule_wait_duration", label_selectors=['name=~"sched-yatp-pool.*"'], ), legend_format="mean-yatp-{{instance}}", ), target( expr=expr_simple( "tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds", ), legend_format="mean-tokio-{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="Async resolve lock queue flow OPS", description="Async resolve lock task dispatch, inqueue, and dequeue throughput.", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_async_resolve_lock_dispatch_total", by_labels=["result"], ), legend_format="dispatch-{{result}}", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_scheduler_async_resolve_lock_inqueue_total", by_labels=["type"], ), legend_format="inqueue-{{type}}", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_scheduler_async_resolve_lock_dequeue_total", by_labels=["type"], ), legend_format="dequeue-{{type}}", additional_groupby=True, ), ], ), graph_panel( title="Async resolve lock queue size", description="Current size of async resolve lock pending, inflight, and blocked queues.", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum( "tikv_scheduler_async_resolve_lock_queue_size", by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.half_row( [ graph_panel( title="Async resolve lock background schedule QPS", description="Async resolve lock background scheduled task QPS, scheduled txn-status QPS, and dropped task QPS.", yaxes=yaxes( left_format=UNITS.OPS_PER_SEC, right_format=UNITS.OPS_PER_SEC ), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_async_resolve_lock_bg_schedule_total", by_labels=["type"], ), additional_groupby=True, ), ], series_overrides=[ series_override( alias="/txn/", yaxis=2, ), ], ), ] ) return layout.row_panel def SchedulerWorkerPool() -> RowPanel: layout = Layout(title="Scheduler Worker Pool") # Row 1: Wait/Schedule Duration + Running Tasks layout.row( [ graph_panel( title="Schedule Wait Duration", description="Time from task spawn to first poll", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_yatp_pool_schedule_wait_duration", label_selectors=['name=~"sched-yatp-pool.*"'], ), legend_format="mean-yatp-{{instance}}", ), target( expr=expr_simple( "tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds", ), legend_format="mean-tokio-{{instance}}", ), ], ), graph_panel( title="Running Tasks", description="Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)", targets=[ target( expr=expr_sum_aggr_over_time( "tikv_scheduler_pool_running_tasks", "avg", "1m", by_labels=["instance"], ), additional_groupby=True, ), ], ), ] ) # Row 2: Poll/Execution Duration layout.row( [ graph_panel( title="Task Poll Duration", description="Time tasks spend executing (being polled)", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_yatp_task_poll_duration", label_selectors=['name=~"sched-yatp-pool.*"'], ), legend_format="mean-yatp-{{instance}}", ), target( expr=expr_simple( "tikv_scheduler_tokio_pool_mean_poll_duration_seconds", ), legend_format="mean-tokio-{{instance}}", ), ], ), graph_panel( title="Task Scheduled Duration", description="Time tasks wait to be polled after being woken", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_simple( "tikv_scheduler_tokio_pool_mean_scheduled_duration_seconds", ), legend_format="mean-tokio-{{instance}}", ), ], ), ] ) # Row 3: Idle Duration and First Poll Count layout.row( [ graph_panel( title="Task Idle Duration", description="Time tasks spend idle waiting on external events (I/O)", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_simple( "tikv_scheduler_tokio_pool_mean_idle_duration_seconds", ), legend_format="mean-tokio-{{instance}}", ), ], ), graph_panel( title="First Poll Count", description="Rate of tasks polled for first time", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_scheduler_tokio_pool_first_poll_total", ), legend_format="tokio-{{instance}}", ), ], ), ] ) return layout.row_panel def GC() -> RowPanel: layout = Layout(title="GC") layout.row( [ graph_panel( title="GC tasks", description="The count of GC tasks processed by gc_worker", targets=[ target( expr=expr_sum_rate( "tikv_gcworker_gc_tasks_vec", by_labels=["task"], ), legend_format="total-{{task}}", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_storage_gc_skipped_counter", by_labels=["task"], ), legend_format="skipped-{{task}}", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gcworker_gc_task_fail_vec", by_labels=["task"], ), legend_format="failed-{{task}}", additional_groupby=True, ), ], ), graph_panel_histogram_quantiles( title="GC tasks duration", description="The time consumed when executing GC tasks", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_gcworker_gc_task_duration_vec", label_selectors=['type="$command"'], hide_count=True, ), ] ) layout.row( [ graph_panel( title="TiDB GC seconds", description="The GC duration", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 1, "tidb_tikvclient_gc_seconds", by_labels=[ "instance", "stage", "keyspace_name", ], skip_default_instance=True, use_shared_pool=True, ), legend_format="{{instance}}-{{keyspace_name}}-{{stage}}", ), ], ), graph_panel( title="TiDB GC worker actions", description="The count of TiDB GC worker actions", targets=[ target( expr=expr_sum_rate( "tidb_tikvclient_gc_worker_actions_total", by_labels=["type", "keyspace_name"], ) .skip_default_instance_selector() .use_shared_pool_selector(), additional_groupby=True, legend_format="{{keyspace_name}}-{{type}}", ), ], ), ] ) layout.row( [ graph_panel( title="ResolveLocks Progress", description="Progress of ResolveLocks, the first phase of GC", targets=[ target( expr=expr_max( "tidb_tikvclient_range_task_stats", label_selectors=['type=~"resolve-locks.*"'], by_labels=["result", "type", "keyspace_name"], ) .skip_default_instance_selector() .use_shared_pool_selector(), additional_groupby=True, legend_format="{{keyspace_name}} {{type}} {{result}}", ), ], ), graph_panel( title="TiKV Auto GC Progress", description="Progress of TiKV's GC", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_sum( "tikv_gcworker_autogc_processed_regions", label_selectors=['type="scan"'], ), "/", expr_sum( "tikv_raftstore_region_count", label_selectors=['type="region"'], ), ), legend_format="{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="GC speed", description="keys / second", targets=[ target( expr=expr_sum_rate( "tikv_storage_mvcc_gc_delete_versions_sum", by_labels=["key_mode"], ), legend_format="{{key_mode}}_keys/s", additional_groupby=True, ), ], ), graph_panel( title="TiKV Side GC Safe Points", description="The GC safe points that has been fetched by TiKV", yaxes=yaxes(left_format=UNITS.DATE_TIME_ISO), null_point_mode=NULL_AS_NULL, targets=[ target( expr=expr_max( "tikv_gcworker_autogc_safe_point", by_labels=["instance", "keyspace_name"], ) .extra("/ (2^18) != 0") .skip_default_instance_selector() .use_shared_pool_selector(), legend_format="{{instance}}-null_keyspace", additional_groupby=True, ), target( expr=expr_max( "tikv_raftstore_keyspace_gc_safe_points", by_labels=["instance", "keyspace_name"], ) .extra("/ (2^18) != 0") .skip_default_instance_selector() .use_shared_pool_selector(), legend_format="{{instance}}-{{keyspace_name}}", additional_groupby=True, ), ], ), ] ) layout.half_row( [ graph_panel( title="GC lifetime", description="The lifetime of TiDB GC", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_max( "tidb_tikvclient_gc_config", label_selectors=['type="tikv_gc_life_time"'], by_labels=["keyspace_name"], ) .skip_default_instance_selector() .use_shared_pool_selector(), legend_format=r"{{keyspace_name}}", ), ], legend=graph_legend( max=False, current=True, hide_zero=False, hide_empty=True ), ), graph_panel( title="GC interval", description="The interval of TiDB GC", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_max( "tidb_tikvclient_gc_config", label_selectors=['type="tikv_gc_run_interval"'], by_labels=["keyspace_name"], ) .skip_default_instance_selector() .use_shared_pool_selector(), legend_format=r"{{keyspace_name}}", ), ], legend=graph_legend( max=False, current=True, hide_zero=False, hide_empty=True ), ), ] ) layout.half_row( [ graph_panel( title="GC in Compaction Filter", description="Keys handled in GC compaction filter", targets=[ target( expr=expr_sum_rate( "tikv_gc_compaction_filtered", by_labels=["key_mode"], ), legend_format="{{key_mode}}_filtered", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_filter_skip", by_labels=["key_mode"], ), legend_format="{{key_mode}}_skipped", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_mvcc_rollback", by_labels=["key_mode"], ), legend_format="{{key_mode}}_mvcc-rollback/mvcc-lock", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_filter_orphan_versions", by_labels=["key_mode"], ), legend_format="{{key_mode}}_orphan-versions", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_filter_perform", by_labels=["key_mode"], ), legend_format="{{key_mode}}_performed-times", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_failure", by_labels=["key_mode", "type"], ), legend_format="{{key_mode}}_failure-{{type}}", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_filter_mvcc_deletion_met", by_labels=["key_mode"], ), legend_format="{{key_mode}}_mvcc-deletion-met", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_filter_mvcc_deletion_handled", by_labels=["key_mode"], ), legend_format="{{key_mode}}_mvcc-deletion-handled", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_gc_compaction_filter_mvcc_deletion_wasted", by_labels=["key_mode"], ), legend_format="{{key_mode}}_mvcc-deletion-wasted", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="GC scan write details", description="GC scan write details", targets=[ target( expr=expr_sum_rate( "tikv_gcworker_gc_keys", label_selectors=['cf="write"'], by_labels=["key_mode", "tag"], ), additional_groupby=True, ), ], ), graph_panel( title="GC scan default details", description="GC scan default details", targets=[ target( expr=expr_sum_rate( "tikv_gcworker_gc_keys", label_selectors=['cf="default"'], by_labels=["key_mode", "tag"], ), additional_groupby=True, ), ], ), ] ) return layout.row_panel def Snapshot() -> RowPanel: layout = Layout(title="Snapshot") layout.row( [ graph_panel( title="Rate snapshot message", description="The rate of Raft snapshot messages sent", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_delta( "tikv_raftstore_raft_sent_message_total", range_selector="1m", label_selectors=['type="snapshot"'], ), ), ], ), graph_panel( title="Snapshot state count", description="The number of snapshots in different states", targets=[ target( expr=expr_sum( "tikv_raftstore_snapshot_traffic_total", by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Snapshot generation/apply wait duration " + OPTIONAL_QUANTILE_INPUT, description="The time snapshot generation/apply tasks spent waiting to be executed.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_snapshot_generation_wait_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-generate-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_snapshot_apply_wait_duration_seconds", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-apply-" + OPTIONAL_QUANTILE_INPUT, ), ], ), graph_panel( title="99% Handle snapshot duration", description="The time consumed when handling snapshots", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_server_send_snapshot_duration_seconds", is_optional_quantile=True, ), legend_format="send-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_snapshot_duration_seconds", label_selectors=['type="apply"'], is_optional_quantile=True, ), legend_format="apply-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_raftstore_snapshot_duration_seconds", label_selectors=['type="generate"'], is_optional_quantile=True, ), legend_format="generate-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="99.99% Snapshot size", description="The snapshot size (P99.99).9999", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_histogram_quantile( 0.9999, "tikv_snapshot_size", ), legend_format="size", additional_groupby=True, ), ], ), graph_panel( title="99.99% Snapshot KV count", description="The number of KV within a snapshot in .9999", targets=[ target( expr=expr_histogram_quantile( 0.9999, "tikv_snapshot_kv_count", ), legend_format="count", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Snapshot Actions", description="Action stats for snapshot generating and applying", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_delta( "tikv_raftstore_snapshot_total", range_selector="1m", by_labels=["type", "status"], ), additional_groupby=True, ), target( expr=expr_sum_delta( "tikv_raftstore_clean_region_count", range_selector="1m", by_labels=["type", "status"], ), legend_format="clean-region-by-{{type}}", additional_groupby=True, ), target( expr=expr_sum_delta( "tikv_server_snapshot_task_total", range_selector="1m", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), graph_panel( title="Snapshot transport speed", description="The speed of sending or receiving snapshot", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_snapshot_limit_transport_bytes", by_labels=["instance", "type"], ) ), target( expr=expr_sum_rate( "tikv_snapshot_limit_generate_bytes", by_labels=["instance", "type"], ), legend_format="{{instance}}-generate-{{type}}", ), ], ), ] ) layout.row( [ graph_panel( title="Snapshot pending applies", description="The number of snapshots waiting to be applied", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_simple( "tikv_raftstore_snapshot_pending_applies", ), legend_format="{{instance}}", ), ], ), ] ) return layout.row_panel def Task() -> RowPanel: layout = Layout(title="Task") layout.row( [ graph_panel( title="Worker handled tasks", description="The number of tasks handled by worker", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_handled_task_total", by_labels=["name"], ), additional_groupby=True, ), ], ), graph_panel( title="Worker pending tasks", description="Current pending and running tasks of worker", targets=[ target( expr=expr_sum( "tikv_worker_pending_task_total", by_labels=["name"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="FuturePool handled tasks", description="The number of tasks handled by future_pool", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_futurepool_handled_task_total", by_labels=["name"], ), additional_groupby=True, ), ], ), graph_panel( title="FuturePool pending tasks", description="Current pending and running tasks of future_pool", targets=[ target( expr=expr_sum_aggr_over_time( "tikv_futurepool_pending_task_total", "avg", range_selector="1m", by_labels=["name"], ), additional_groupby=True, ), ], ), ] ) return layout.row_panel def CoprocessorOverview() -> RowPanel: layout = Layout(title="Coprocessor Overview") layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Request duration", heatmap_description="The time consumed to handle coprocessor read requests", graph_title="Request duration", graph_description="The time consumed to handle coprocessor read requests", yaxis_format=UNITS.SECONDS, metric="tikv_coprocessor_request_duration_seconds", graph_by_labels=["req"], ), ) layout.row( [ graph_panel( title="Coprocessor CPU Usage per Keyspace", description="Average coprocessor CPU cores used by each keyspace over the recent window", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum( "tikv_coprocessor_keyspace_handle_cpu_cores", by_labels=["instance", "keyspace_id"], ), legend_format="{{instance}}-{{keyspace_id}}", ), ], ), ] ) layout.row( [ graph_panel( title="Total Requests", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_request_duration_seconds_count", by_labels=["req"], ), additional_groupby=True, ), ], ), graph_panel( title="Total Request Errors", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_request_error", by_labels=["reason"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="KV Cursor Operations", targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_scan_keys_sum", by_labels=["req"], ), additional_groupby=True, ), ], ), graph_panel_histogram_quantiles( title="KV Cursor Operations", description="", metric="tikv_coprocessor_scan_keys", yaxes=yaxes(left_format=UNITS.SHORT), by_labels=["req"], hide_count=True, ), ] ) layout.row( [ graph_panel( title="Total RocksDB Perf Statistics", targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_rocksdb_perf", label_selectors=['metric="internal_delete_skipped_count"'], by_labels=["req"], ), legend_format="delete_skipped-{{req}}", additional_groupby=True, ), ], ), graph_panel( title="Total Response Size", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_response_bytes", ), ), ], ), ] ) layout.row( [ graph_panel( title="Memory Quota", description="Total bytes of memory used by coprocessor requests", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_coprocessor_memory_quota", by_labels=["instance", "type"], ), ), ], ) ] ) return layout.row_panel def CoprocessorDetail() -> RowPanel: layout = Layout(title="Coprocessor Detail") layout.row( [ graph_panel_histogram_quantiles( title="Handle duration", description="The time consumed when handling coprocessor requests", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_coprocessor_request_handle_seconds", by_labels=["req"], hide_avg=True, hide_count=True, ), graph_panel_histogram_quantiles( title="Handle duration by store", description="The time consumed to handle coprocessor requests per TiKV instance", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_coprocessor_request_handle_seconds", by_labels=["req", "instance"], hide_avg=True, hide_count=True, additional_groupby=False, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Wait duration", description="The time consumed when coprocessor requests are wait for being handled", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_coprocessor_request_wait_seconds", label_selectors=['type="all"'], by_labels=["req"], hide_avg=True, hide_count=True, ), graph_panel_histogram_quantiles( title="Wait duration by store", description="The time consumed when coprocessor requests are wait for being handled in each TiKV instance", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_coprocessor_request_wait_seconds", label_selectors=['type="all"'], by_labels=["req", "instance"], hide_avg=True, hide_count=True, additional_groupby=False, ), ] ) layout.row( [ graph_panel( title="Total DAG Requests", targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_dag_request_count", by_labels=["vec_type"], ), additional_groupby=True, ), ], ), graph_panel( title="Total DAG Executors", description="The total number of DAG executors", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_executor_count", by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Remote Copr Offload Reasons", description="The QPS of remote execution decisions by primary reason: size/range eligibility, long-query or lazy-pattern eligibility, load-aware force thresholds, or CPU state", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_remote_decision_total", label_selectors=['result="remote"'], by_labels=["reason"], ), legend_format="{{reason}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Total Ops Details (Table Scan)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_scan_details", label_selectors=[ 'req=~"select|select_by_in_memory_engine"' ], by_labels=["tag"], ), additional_groupby=True, ), ], ), graph_panel( title="Total Ops Details (Index Scan)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_scan_details", label_selectors=['req=~"index|index_by_in_memory_engine"'], by_labels=["tag"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Total Ops Details by CF (Table Scan)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_scan_details", label_selectors=[ 'req=~"select|select_by_in_memory_engine"' ], by_labels=["cf", "tag"], ), additional_groupby=True, ), ], ), graph_panel( title="Total Ops Details by CF (Index Scan)", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_rate( "tikv_coprocessor_scan_details", label_selectors=['req=~"index|index_by_in_memory_engine"'], by_labels=["cf", "tag"], ), additional_groupby=True, ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Memory lock checking duration", heatmap_description="The time consumed on checking memory locks for coprocessor requests", graph_title="Memory lock checking duration", graph_description="The time consumed on checking memory locks for coprocessor requests", yaxis_format=UNITS.SECONDS, metric="tikv_coprocessor_mem_lock_check_duration_seconds", ), ) return layout.row_panel def InMemoryEngine() -> RowPanel: layout = Layout(title="In Memory Engine") layout.row( [ graph_panel( title="OPS", description="Operation per second for cf", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_in_memory_engine_kv_operations", by_labels=["instance", "type"], ), legend_format="{{type}}-{{instance}}", additional_groupby=True, ), ], ), graph_panel( title="Read MBps", description="The total bytes of read in RocksDB and in-memory engine(the same with panel Cluster/MBps for read)", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_engine_flow_bytes", label_selectors=['type=~"bytes_read|iter_bytes_read"'], ), legend_format=r"rocksdb-{{instance}}", ), target( expr=expr_sum_rate( "tikv_in_memory_engine_flow", label_selectors=['type=~"bytes_read|iter_bytes_read"'], ), legend_format=r"in-memory-engine-{{instance}}", ), ], ), graph_panel_histogram_quantiles( title="Coprocessor Handle duration", description="The time consumed when handling coprocessor requests", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_coprocessor_request_handle_seconds", by_labels=["req"], hide_avg=True, hide_count=True, ), ] ) layout.row( [ graph_panel( title="Region Cache Hit", description="Count of region cache hit", targets=[ target( expr=expr_sum_rate( "tikv_snapshot_type_count", label_selectors=['type="in_memory_engine"'], by_labels=["instance"], ), legend_format="count-{{instance}}", ), ], ), graph_panel( title="Region Cache Hit Rate", description="Region cache hit rate", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_sum_rate( "tikv_snapshot_type_count", label_selectors=['type="in_memory_engine"'], by_labels=["instance"], ), "/", expr_sum_rate( "tikv_snapshot_type_count", by_labels=["instance"], ), ), legend_format="rate-{{instance}}", additional_groupby=True, ), ], ), graph_panel( title="Region Cache Miss Reason", description="Reasons for region cache miss", targets=[ target( expr=expr_sum_rate( "tikv_in_memory_engine_snapshot_acquire_failed_reason_count", by_labels=["instance", "type"], ), legend_format="{{type}}-{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="Memory Usage", description="The memory usage of the in-memory engine", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_avg( "tikv_in_memory_engine_memory_usage_bytes", by_labels=["instance"], ), ), ], ), graph_panel( title="Region Count", description="The count of different types of region", targets=[ target( expr=expr_avg( "tikv_in_memory_engine_cache_count", by_labels=["instance", "type"], ), legend_format="{{instance}}--{{type}}", ), ], ), ] ) layout.row( [ graph_panel( title="GC Filter", description="Rang cache engine garbage collection information", targets=[ target( expr=expr_sum_rate( "tikv_in_memory_engine_gc_filtered", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), heatmap_panel( title="Region GC Duration", description="The handle duration of region gc", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_in_memory_engine_gc_duration_secs_bucket", ), ] ) layout.row( [ heatmap_panel( title="Region Load Duration", description="The handle duration of region load", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_in_memory_engine_load_duration_secs_bucket", ), graph_panel( title="Region Load Count", description="The count of region loading per seconds", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_delta( "tikv_in_memory_engine_load_duration_secs_count", by_labels=["instance"], ), legend_format="{{instance}}", ), ], ), ] ) layout.row( [ heatmap_panel( title="Region Eviction Duration", description="The handle duration of region eviction", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_in_memory_engine_eviction_duration_secs_bucket", ), graph_panel( title="Region Eviction Count", description="The count of region eviction per seconds", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_delta( "tikv_in_memory_engine_eviction_duration_secs_count", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Write duration", heatmap_description="The time consumed of write in region cache engine", graph_title="99% In-memory engine write duration per server", graph_description="The time consumed of write in region cache engine per TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_in_memory_engine_write_duration_seconds", ) ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Prepare for write duration", heatmap_description="The time consumed of prepare for write in the in-memory engine", graph_title="99% In-memory engine prepare for write duration per server", graph_description="The time consumed of prepare for write in the in-memory engine per TiKV instance", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="tikv_in_memory_engine_prepare_for_write_duration_seconds", ) ) layout.row( [ graph_panel( title="Iterator operations", description="The count of different type of iteration operations", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_in_memory_engine_locate", label_selectors=[ 'type="number_db_seek"', ], by_labels=[], # override default by instance. ), legend_format="seek", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_in_memory_engine_locate", label_selectors=[ 'type="number_db_seek_found"', ], by_labels=[], # override default by instance. ), legend_format="seek_found", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_in_memory_engine_locate", label_selectors=[ 'type="number_db_next"', ], by_labels=[], # override default by instance. ), legend_format="next", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_in_memory_engine_locate", label_selectors=[ 'type="number_db_next_found"', ], by_labels=[], # override default by instance. ), legend_format="next_found", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_in_memory_engine_locate", label_selectors=[ 'type="number_db_prev"', ], by_labels=[], # override default by instance. ), legend_format="prev", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_in_memory_engine_locate", label_selectors=[ 'type="number_db_prev_found"', ], by_labels=[], # override default by instance. ), legend_format="prev_found", additional_groupby=True, ), ], ), graph_panel( title="Seek duration", description="The time consumed when executing seek operation", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 1, "tikv_in_memory_engine_seek_duration", ), legend_format="max", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_in_memory_engine_seek_duration", is_optional_quantile=True, ), legend_format=OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.95, "tikv_in_memory_engine_seek_duration", ), legend_format="95%", additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_in_memory_engine_seek_duration", by_labels=["type"], ), legend_format="avg", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Oldest Auto GC SafePoint", description="Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one", yaxes=yaxes(left_format=UNITS.DATE_TIME_ISO), targets=[ target( expr=expr_max( "tikv_in_memory_engine_oldest_safe_point", ) .extra("/ (2^18)") .skip_default_instance_selector(), additional_groupby=True, ), ], ), graph_panel( title="Newest Auto GC SafePoint", description="Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one", yaxes=yaxes(left_format=UNITS.DATE_TIME_ISO), targets=[ target( expr=expr_max( "tikv_in_memory_engine_newest_safe_point", ) .extra("/ (2^18)") .skip_default_instance_selector(), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Auto GC SafePoint Gap", description="The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine", yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=expr_operator( expr_sum( "tikv_in_memory_engine_newest_safe_point", ) .extra("/ (2^18)") .skip_default_instance_selector(), "-", expr_sum( "tikv_in_memory_engine_oldest_safe_point", ) .extra("/ (2^18)") .skip_default_instance_selector(), ), additional_groupby=True, legend_format="{{instance}}", ), ], ), graph_panel( title="Auto GC SafePoint Gap With TiKV", description="The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point", yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=expr_max( "tikv_safe_point_gap_with_in_memory_engine", ) .extra("/ (2^18)") .skip_default_instance_selector(), additional_groupby=True, ), ], ), ] ) return layout.row_panel def Threads() -> RowPanel: layout = Layout(title="Threads") layout.row( [ graph_panel( title="Threads state", targets=[ target( expr=expr_sum( "tikv_threads_state", by_labels=["instance", "state"], ), ), target( expr=expr_sum( "tikv_threads_state", by_labels=["instance"], ), legend_format="{{instance}}-total", ), ], ), graph_panel( title="Threads IO", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_topk( 20, "%s" % expr_sum_rate( "tikv_threads_io_bytes_total", by_labels=["name", "io"], ).extra("> 1024"), ), legend_format="{{name}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Thread Voluntary Context Switches", targets=[ target( expr=expr_topk( 20, "%s" % expr_max_rate( "tikv_thread_voluntary_context_switches", by_labels=["name"], ).extra("> 100"), ), legend_format="{{name}}", ), ], ), graph_panel( title="Thread Nonvoluntary Context Switches", targets=[ target( expr=expr_topk( 20, "%s" % expr_max_rate( "tikv_thread_nonvoluntary_context_switches", by_labels=["name"], ).extra("> 100"), ), legend_format="{{name}}", ), ], ), ] ) return layout.row_panel def KvEngine() -> RowPanel: layout = Layout(title="Kv Engine") layout.row( [ graph_panel( title="Memory Usage", description="The memory usage of Kv Engine", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_max( "kv_engine_mem_size_bytes", label_selectors=['db="kv"'], # only select `kv` db by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_max( "kv_engine_mem_size_quota_bytes", by_labels=[], # override default by instance. ), hide=True, legend_format="memtable-quota", additional_groupby=True, ), target( expr=expr_max( "kv_engine_dfs_load_memory_usage_bytes", by_labels=[], # override default by instance. ), legend_format="dfs_load", additional_groupby=True, ), target( expr=expr_max( "kv_engine_ia_manager_segments_memory_size", by_labels=[], # override default by instance. ), legend_format="segment", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Get operations", description="The count of get operations", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_get_duration_seconds_count", by_labels=[], # override default by instance. ), legend_format="get", additional_groupby=True, ), ], ), graph_panel( title="Get duration", description="The time consumed when executing get operations", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.9999, "kv_engine_get_duration_seconds" ), legend_format="99.99%", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "kv_engine_get_duration_seconds", is_optional_quantile=True, ), legend_format=OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_avg( "kv_engine_get_duration_seconds", by_labels=[], # override default by instance. ), legend_format="avg", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Seek operations", description="The count of seek operations", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_seek_duration_seconds_count", # override default by instance. by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Seek duration", description="The time consumed when executing seek operation", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.9999, "kv_engine_seek_duration_seconds" ), legend_format="99.99%", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "kv_engine_seek_duration_seconds", is_optional_quantile=True, ), legend_format=OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_avg( "kv_engine_seek_duration_seconds", by_labels=[], # override default by instance. ), legend_format="avg", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Write operations", description="The count of write operations", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_write_duration_seconds_count", by_labels=[], # override default by instance. ), legend_format="write", additional_groupby=True, ), ], ), graph_panel( title="Write duration", description="The time consumed when executing write operation", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.9999, "kv_engine_write_duration_seconds" ), legend_format="99.99%", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "kv_engine_write_duration_seconds", is_optional_quantile=True, ), legend_format=OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_avg( "kv_engine_write_duration_seconds", by_labels=[], # override default by instance. ), legend_format="avg", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Write flow", description="The bytes flow of write operations", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "kv_engine_write_flow_sum", label_selectors=[ 'type="bytes"', ], by_labels=[], # override default by instance. ), legend_format="write", additional_groupby=True, ), ], ), graph_panel_histogram_quantiles( title="Bytes / Write", description="The bytes per write", yaxes=yaxes(left_format=UNITS.BYTES_IEC), metric="kv_engine_write_flow", label_selectors=['type="bytes"'], hide_count=True, ), ] ) layout.row( [ graph_panel( title="Keys flow", description="The bytes flow of write operations", yaxes=yaxes(left_format=UNITS.COUNTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_write_flow_sum", label_selectors=[ 'type="keys"', ], by_labels=[], # override default by instance. ), legend_format="write", additional_groupby=True, ), ], ), graph_panel_histogram_quantiles( title="Keys / Write", description="The count of keys per write", yaxes=yaxes(left_format=UNITS.NO_FORMAT), metric="kv_engine_write_flow", label_selectors=['type="keys"'], hide_count=True, ), ] ) layout.row( [ graph_panel( title="Block Cache QPS", description="The lookup and miss QPS of block cache", yaxes=yaxes(left_format=UNITS.COUNTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_cache_lookup", ), legend_format="lookup-{{instance}}", ), target( expr=expr_sum_rate( "kv_engine_cache_miss", ), legend_format="miss-{{instance}}", ), ], ), graph_panel( title="Block Cache Miss Rate", description="The ratio of block cache misses to lookups", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_sum_rate("kv_engine_cache_miss"), "/", expr_sum_rate("kv_engine_cache_lookup"), ), legend_format="miss rate-{{instance}}", ), ], ), graph_panel( title="Shards", description="The number of shards in KV Engine", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_shards_total", by_labels=[], # override default by instance. ), legend_format="total", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_initial_flushed_shards", by_labels=[], # override default by instance. ), legend_format="initial_flushed", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_active_shards", by_labels=[], # override default by instance. ), legend_format="active", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_compacting_shards", by_labels=[], # override default by instance. ), legend_format="compacting", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_pending_compaction_shards", by_labels=[], # override default by instance. ), legend_format="pending_compaction", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_has_del_prefixes_shards", by_labels=[], # override default by instance. ), legend_format="has_del_prefixes", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_inner_key_shards", by_labels=[], # override default by instance. ), hide=True, legend_format="inner_key", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_ready_destroy_range_shards", by_labels=[], # override default by instance. ), legend_format="ready_destroy_range", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Files Count", description="The number of files in KV Engine", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_cf_level_num_files", by_labels=["cf", "level"], ), additional_groupby=True, ), ], ), graph_panel( title="Files Size", description="The size of files in KV Engine", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "kv_engine_cf_level_total_sizes_bytes", by_labels=["cf", "level"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Memtable Count", description="The count of memtables", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_mem_tables_count", by_labels=[], # override default by instance. ), additional_groupby=True, ), ], ), graph_panel( title="Memtable Size", description="The size of memtables", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "kv_engine_mem_size_bytes", label_selectors=[ 'db="kv"', 'type="memtable"', ], # only select `kv` db ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="L0 Table Count", description="The count of L0 tables", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_l0_tables_count", by_labels=[], # override default by instance. ), additional_groupby=True, ), ], ), graph_panel( title="L0 Table Size", description="The size of L0 tables", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "kv_engine_l0_tables_size_bytes", ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Blob Table Count", description="The count of blob tables", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_blob_tables_count", by_labels=[], # override default by instance. ), additional_groupby=True, ), ], ), graph_panel( title="Blob Table Size", description="The size of blob tables", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "kv_engine_in_use_blob_size_bytes", ), legend_format="in_use-{{instance}}", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_total_blob_size_bytes", ), legend_format="total-{{instance}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Partial File Count", description="The count of partial files", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_partial_l0_count", by_labels=[], # override default by instance. ), legend_format="partial_l0", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_partial_blob_count", by_labels=[], # override default by instance. ), legend_format="partial_blob", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_partial_ln_count", by_labels=[], # override default by instance. ), legend_format="partial_ln", additional_groupby=True, ), ], ), graph_panel( title="Index & Filter Size", description="The size of index and filter files", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "kv_engine_index_size_bytes", by_labels=[], # override default by instance. ), legend_format="index", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_filter_size_bytes", by_labels=[], # override default by instance. ), legend_format="filter", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Entries Count", description="The count of entries", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "kv_engine_entries", by_labels=[], # override default by instance. ), legend_format="entries", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_old_entries", by_labels=[], # override default by instance. ), legend_format="old_entries", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_tombstones", by_labels=[], # override default by instance. ), legend_format="tombstones", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="DFS Throughput", description="The throughput of DFS operations", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "kv_engine_dfs_throughput_bytes", by_labels=["type"], # Currently, only S3 is supported. # LocalFs is unnecessary since it is only used for testing, # and other types are not supported yet. label_selectors=['dfs_type="s3"'], ), legend_format="{{type}}", additional_groupby=True, ), ], ), graph_panel( title="Level Write Throughput", description="The throughput of load file at each level", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "kv_engine_level_write_bytes", by_labels=["level"], ), legend_format="{{level}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="DFS Request Rate", description="The OPS of DFS operations", yaxes=yaxes(left_format=UNITS.COUNTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_dfs_request_counter", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), graph_panel( title="DFS Request Retry Rate", description="The OPS of DFS request retries", yaxes=yaxes(left_format=UNITS.COUNTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_dfs_retry_counter", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="DFS Request Duration", description="The duration of DFS operations in ms", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="kv_engine_dfs_latency_seconds", hide_count=True, ), graph_panel_histogram_quantiles( title="DFS Request Duration with Retry", description="The duration of DFS operations including retry times in ms", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="kv_engine_dfs_latency_with_retry_seconds", hide_count=True, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="DFS Load Memory Wait Duration", description="Wait duration when acquiring DFS load memory (only recorded when memory is not immediately available)", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="kv_engine_dfs_load_memory_wait_duration_seconds", hide_count=True, ), graph_panel( title="Prepare ChangeSet File Load Source", description="The file load statistic when preparing changeset", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_prepare_load_remote_file", by_labels=[], ), legend_format="s3", additional_groupby=True, ), target( expr=expr_sum_rate( "kv_engine_prepare_use_local_file", by_labels=[], ), legend_format="local", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="CacheFS Hit", description="Count of CacheFS hit", targets=[ target( expr=expr_sum_rate( "kv_engine_cachefs_req_count", label_selectors=['type="hit"'], by_labels=["instance"], ), legend_format="count-{{instance}}", ), ], ), graph_panel( title="CacheFS Hit Rate", description="CacheFS hit rate", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_sum_rate( "kv_engine_cachefs_req_count", label_selectors=['type="hit"'], by_labels=["instance"], ), "/", expr_sum_rate( "kv_engine_cachefs_req_count", by_labels=["instance"], ), ), legend_format="rate-{{instance}}", additional_groupby=True, ), ], ), graph_panel( title="Duplicated ChangeSet Rate", description="The rate of duplicated changesets detected by KvEngine", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_duplicated_changeset_counter", by_labels=[], ), legend_format="duplicated changeset", additional_groupby=True, ), ], ), ] ) layout.row( [ heatmap_panel( title="Ingest Level Distribution", description="The distribution of ingestion levels (0-3) in the storage engine", metric="kv_engine_ingest_level_bucket", y_axis=yaxis(format=UNITS.SHORT, decimals=1), ), graph_panel( title="Ingest Level Rate", description="The rate of ingestion operations by level", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_ingest_level_count", by_labels=[], # override default by instance. ), legend_format="total ingestion rate", additional_groupby=True, ), ], ), ] ) return layout.row_panel def IA() -> RowPanel: layout = Layout(title="IA") layout.row( [ graph_panel( title="IA Segment Memory Cache", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_max( "kv_engine_ia_manager_segments_memory_size", ), legend_format="used-{{instance}}", ), target( expr=expr_max( "kv_engine_ia_small_queue_capacity", ), legend_format="cap-{{instance}}", ), ], ), graph_panel( title="IA Segment Disk Cache", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_max( "kv_engine_ia_manager_segments_disk_size", ), legend_format="used-{{instance}}", ), target( expr=expr_max( "kv_engine_ia_main_queue_capacity", ), legend_format="cap-{{instance}}", ), ], ), ] ) layout.row( [ graph_panel( title="IA Segment Cache Hit", description="Count of IA segment hit", targets=[ target( expr=expr_sum_rate( "kv_engine_ia_remote_read_segment_cache_hit", ), legend_format="count-{{instance}}", ), ], ), graph_panel( title="IA Segment Cache Hit Rate", description="IA Segment hit rate", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_sum_rate( "kv_engine_ia_remote_read_segment_cache_hit", by_labels=["instance"], ), "/", expr_operator( expr_sum_rate( "kv_engine_ia_remote_read_segment_cache_hit", by_labels=["instance"], ), "+", expr_sum_rate( "kv_engine_ia_remote_read_segment_cache_miss", by_labels=["instance"], ), ), ), legend_format="rate-{{instance}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="IA IO Throughput", description="Throughput through IA IO rate limiter and configured cap. Cap metrics are per TiKV process, summed by this panel; defaults are 500 MiB/s per process, and 0 means unlimited.", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "kv_engine_ia_io_throughput_bytes", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_ia_io_throughput_cap", by_labels=["type"], ), legend_format="cap-{{type}}", additional_groupby=True, ), ], ), graph_panel_histogram_quantiles( title="IA IO Throttle Duration", description="Time spent throttling IA remote reads and local writes.", yaxes=yaxes(left_format=UNITS.SECONDS), metric="kv_engine_ia_io_throttle_duration_seconds", by_labels=["type"], hide_count=True, ), ] ) layout.row( [ graph_panel( title="Storage Class Transition Regions", description="Pending and inflight storage-class transition regions. Limits are per TiKV process.", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum( "kv_engine_ia_transition_region_pending", by_labels=[], ), legend_format="pending", additional_groupby=True, ), target( expr=expr_sum( "kv_engine_ia_transition_region_inflight", by_labels=[], ), legend_format="inflight", additional_groupby=True, ), ], ), graph_panel( title="Storage Class Transition Region Result Rate", description="Rate of storage-class transition region completions by result.", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_ia_transition_region_finished_total", by_labels=["result"], ), legend_format="{{result}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="IA to Standard Remote Read Throughput", description="Remote read bytes admitted by the IA to Standard transition limiter. Bytes are based on DFS file size before each remote read starts.", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "kv_engine_ia_to_standard_transition_remote_read_bytes_total", by_labels=[], ), legend_format="admitted", additional_groupby=True, ), ], ), graph_panel( title="IA to Standard Remote Read Throttle Duration", description="Rate of time spent throttling IA to Standard transition remote reads.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_sum_rate( "kv_engine_ia_to_standard_transition_remote_read_throttle_duration_seconds", by_labels=[], ), legend_format="throttled", additional_groupby=True, ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="IA Segmenet Read Duration", heatmap_description="The time taken by KvEngine to complete read a IA segment", graph_title="99% IA segment read duration", graph_description="The time taken by KvEngine to complete read a IA segment", graph_by_labels=["instance"], graph_hides=["count", "avg"], yaxis_format=UNITS.SECONDS, metric="kv_engine_ia_remote_read_segment_duration_seconds", ) ) layout.row( [ graph_panel( title="Storage Size by Storage Class", description="Storage size used by each storage class (IA and Standard)", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_store_size_bytes", label_selectors=['type="used"'], by_labels=["storage_class"], ), legend_format="{{storage_class}}", additional_groupby=True, ), ], ), ], ) return layout.row_panel def KvEngineCompaction() -> RowPanel: layout = Layout(title="Kv Engine - Compaction") layout.row( [ graph_panel( title="Compaction Process Bytes Throughput", description="Bytes processed by compaction (input KV / output compressed)," " broken down by type, kind, and format", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "kv_engine_compaction_process_bytes_total", by_labels=["type", "kind", "format"], ), legend_format="{{type}}-{{kind}}-{{format}}", additional_groupby=True, ), ], ), graph_panel( title="Compaction Process Rows Throughput", description="Rows processed by compaction (input/output)," " broken down by type, kind, and format", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_compaction_process_rows_total", by_labels=["type", "kind", "format"], ), legend_format="{{type}}-{{kind}}-{{format}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Compaction Task Input Bytes Throughput", description="Per-task compaction input throughput (bytes/sec), by type", metric="kv_engine_compaction_task_handled_bytes_throughput", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), by_labels=["type"], hide_avg=True, hide_count=True, ), graph_panel_histogram_quantiles( title="Compaction Task Input Rows Throughput", description="Per-task compaction input throughput (rows/sec), by type", metric="kv_engine_compaction_task_handled_rows_throughput", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), by_labels=["type"], hide_avg=True, hide_count=True, ), ] ) layout.row( [ graph_panel( title="Compaction Result Rate", description="Rate of compaction results (success/failure)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_compaction_result_counter", by_labels=["result"], ), legend_format="{{result}}", additional_groupby=True, ), ], ), graph_panel( title="Compaction Trigger Rate", description="Rate of compaction triggers, by type", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_compaction_trigger_counter", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Remote Compaction Exceed Memory Limit Rate", description="Rate of remote compaction requests that exceed the memory limit", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "kv_engine_remote_compact_exceed_memory_limit_counter", label_selectors=[r'instance=~"${worker_instance:regex}"'], by_labels=[], skip_default_instance=True, ), legend_format="exceed memory limit", additional_groupby=True, ), ], ), ] ) return layout.row_panel def RaftEngine() -> RowPanel: layout = Layout(title="Raft Engine") layout.row( [ graph_panel( title="Resource Usage", description="The resource usage of Raft Engine", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "raft_engine_resouce_usage_bytes", by_labels=["type"], # override default by instance. ), legend_format="{{type}}", additional_groupby=True, ), ], ), graph_panel( title="Entry Count", description="The average number of raft log entries", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_avg( "raft_engine_total_entries_count", by_labels=["type"], ), legend_format="{{type}} entries", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Operation", description="The count of operations per second", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "raft_engine_persist_duration_seconds_count", by_labels=[], # override default by instance. ), legend_format="write", additional_groupby=True, ), target( expr=expr_sum_rate( "raft_engine_fetch_entries_duration_seconds_count", by_labels=[], # override default by instance. ), legend_format="read", additional_groupby=True, ), ], ), graph_panel( title="Read/Write Duration " + OPTIONAL_QUANTILE_INPUT, description="The time used in read/write operation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "raft_engine_persist_duration_seconds", is_optional_quantile=True, ), legend_format="write-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "raft_engine_fetch_entries_duration_seconds", is_optional_quantile=True, ), legend_format="read-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Flow", description="The I/O flow rate", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "raft_engine_region_write_batch_size_sum", by_labels=[], # override default by instance. ), legend_format="write", additional_groupby=True, ), ], ), graph_panel( title="Write Duration Breakdown " + OPTIONAL_QUANTILE_INPUT, description=OPTIONAL_QUANTILE_INPUT + " duration breakdown of write operation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "raft_engine_wal_write_duration_seconds", is_optional_quantile=True, ), legend_format="wal - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.999, "raft_engine_wal_write_throttle_duration_seconds", by_labels=["type"], is_optional_quantile=True, ), legend_format="throttling - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "raft_engine_apply_duration_seconds", is_optional_quantile=True, ), legend_format="apply - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Bytes / Written", description="The bytes per write", yaxes=yaxes(left_format=UNITS.BYTES_IEC), metric="raft_engine_region_write_batch_size", hide_count=True, ), graph_panel( title="Duration of WAL Operation - " + OPTIONAL_QUANTILE_INPUT, description=OPTIONAL_QUANTILE_INPUT + " duration breakdown of WAL write operation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.999, "raft_engine_truncate_duration_seconds" ), legend_format="truncate - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.999, "raft_engine_rotate_duration_seconds" ), legend_format="rotate - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.999, "raft_engine_compact_wal_duration_seconds" ), legend_format="compact_wal - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="File Count", description="The average number of files", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_avg( "raft_engine_pending_compaction_wals", by_labels=[], ), legend_format="pending_compaction", additional_groupby=True, ), target( expr=expr_avg( "raft_engine_total_wals_count", by_labels=[], ), legend_format="total", additional_groupby=True, ), ], ), graph_panel( title="Other Durations " + OPTIONAL_QUANTILE_INPUT, description="The " + OPTIONAL_QUANTILE_INPUT + " duration of operations of raft engine", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), targets=[ target( expr=expr_histogram_quantile( 0.99, "raft_engine_take_snapshot_duration_seconds", is_optional_quantile=True, ), legend_format="snapshot - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "raft_engine_backup_duration_seconds", by_labels=["type"], is_optional_quantile=True, ), legend_format="backup-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Raft Log GC Size", description="The size of GCed raft log", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_avg( "raft_engine_log_file_gc_size", by_labels=["status"], ), additional_groupby=True, ), ], ), graph_panel( title="Backup Count", description="The average number of raft engine backup", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum_delta( "raft_engine_backup_counter", by_labels=["status"], ).extra("> -10"), additional_groupby=True, ), ], ), ] ) layout.row( [ stat_panel( title="DFS Worker Status", targets=[ target( expr=expr_simple("raft_engine_dfs_worker_healthy"), legend_format="{{ instance }}", ), ], mappings=[ StatValueMappings( StatValueMappingItem("Unhealthy", "0", "red"), StatValueMappingItem("Healthy", "1", "green"), ), ], ), ] ) layout.row( [ stat_panel( title="Total Uploaded Bytes to DFS", description="Total number of bytes uploaded to DFS", format=UNITS.BYTES_IEC, targets=[ target( expr=expr_simple("raft_engine_dfs_uploaded_bytes_total"), legend_format="{{ instance }}", ), ], mappings=[], ) ] ) layout.row( [ graph_panel( title="DFS Request Rate", description="The count of DFS requests issued by rfengine.", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum_rate("raft_engine_dfs_requests_total"), additional_groupby=True, ), ], ), graph_panel( title="DFS Upload Rate", description="The byte rates of uploading WAL to DFS.", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate("raft_engine_dfs_uploaded_bytes_total"), additional_groupby=True, ), ], ), ] ) return layout.row_panel def CloudWorkerService( copr_process_selector: str = 'job=~"coprocessor-worker"', tikv_worker_process_selector: str = 'job=~"tikv-worker"', copr_worker_prefetch_selectors: list[str] = ['instance=~"$worker_instance"'], ) -> RowPanel: layout = Layout(title="Cloud Worker Service") copr_cpu_quota = expr_operator( expr_simple( "container_spec_cpu_quota", label_selectors=[copr_process_selector], ), "/", expr_simple( "container_spec_cpu_period", label_selectors=[copr_process_selector], ), ) copr_memory_quota = expr_operator( expr_simple( "container_spec_memory_limit_bytes", label_selectors=[copr_process_selector], ), ">", "0", ) tikv_worker_cpu_quota = expr_operator( expr_simple( "container_spec_cpu_quota", label_selectors=[tikv_worker_process_selector], ), "/", expr_simple( "container_spec_cpu_period", label_selectors=[tikv_worker_process_selector], ), ) tikv_worker_memory_quota = expr_operator( expr_simple( "container_spec_memory_limit_bytes", label_selectors=[tikv_worker_process_selector], ), ">", "0", ) layout.row( [ graph_panel( title="Copr Worker CPU", description="The CPU usage of each copr worker instance", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( "process_cpu_seconds_total", label_selectors=[copr_process_selector], ), ), target( expr=copr_cpu_quota, legend_format=r"quota-{{instance}}", hide=True, ), target( expr=f"max({copr_cpu_quota})", legend_format=r"max-quota", ), target( expr=f"min({copr_cpu_quota})", legend_format=r"min-quota", ), ], ), graph_panel( title="Copr Worker Memory", description="The memory usage per copr worker instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "process_resident_memory_bytes", label_selectors=[copr_process_selector], ), ), target( expr=copr_memory_quota, legend_format=r"quota-{{instance}}", hide=True, ), target( expr=f"max({copr_memory_quota})", legend_format=r"max-quota", ), target( expr=f"min({copr_memory_quota})", legend_format=r"min-quota", ), ], ), ] ) layout.row( [ graph_panel( title="TiKV Worker CPU", description="The CPU usage of each TiKV worker instance", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( "process_cpu_seconds_total", label_selectors=[tikv_worker_process_selector], ), ), target( expr=tikv_worker_cpu_quota, legend_format=r"quota-{{instance}}", hide=True, ), target( expr=f"max({tikv_worker_cpu_quota})", legend_format=r"max-quota", ), target( expr=f"min({tikv_worker_cpu_quota})", legend_format=r"min-quota", ), ], ), graph_panel( title="TiKV Worker Memory", description="The memory usage per TiKV worker instance", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "process_resident_memory_bytes", label_selectors=[tikv_worker_process_selector], ), ), target( expr=tikv_worker_memory_quota, legend_format=r"quota-{{instance}}", hide=True, ), target( expr=f"max({tikv_worker_memory_quota})", legend_format=r"max-quota", ), target( expr=f"min({tikv_worker_memory_quota})", legend_format=r"min-quota", ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Compaction Request duration", heatmap_description="The time consumed to handle remote compaction requests", graph_title="Compaction Request duration", graph_description="The time consumed to handle remote compaction requests", yaxis_format=UNITS.SECONDS, metric="tikv_worker_remote_compact_request_duration_seconds", label_selectors=['instance=~"$worker_instance"'], skip_default_instance=True, ), ) layout.row( [ graph_panel( title="Remote Processing Requests", description="Number of remote compaction or coprocessor requests are under processing", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum( "tikv_worker_remote_compact_processing_requests_counter", label_selectors=['instance=~"$worker_instance"'], by_labels=[], skip_default_instance=True, ), additional_groupby=True, legend_format="compact-processing", ), target( expr=expr_sum( "tikv_worker_remote_cop_processing_requests_counter", label_selectors=['instance=~"$worker_instance"'], by_labels=[], skip_default_instance=True, ), additional_groupby=True, legend_format="copr-processing", ), ], ), graph_panel( title="Remote Requests Failures", description="The number of remote requests that failed, by low-cardinality failure reason", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_remote_compact_failed_requests_counter", label_selectors=['instance=~"$worker_instance"'], by_labels=["type"], skip_default_instance=True, ), additional_groupby=True, legend_format="compact-{{type}}", ), target( expr=expr_sum_rate( "tikv_worker_remote_cop_failed_requests_counter", label_selectors=['instance=~"$worker_instance"'], by_labels=["type"], skip_default_instance=True, ), additional_groupby=True, legend_format="copr-{{type}}", ), ], ), ] ) layout.row( [ graph_panel( title="Concurrency Limiter Waiting Length", description="Number of remote requests being waiting, limited by concurrency limiter", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum( "tikv_worker_limiter_waiting_requests_counter", label_selectors=['instance=~"$worker_instance"'], by_labels=["type"], ), additional_groupby=True, ), ], ), graph_panel( title="Concurrency Limiter Waiting Duration", description="The waiting duration of handling remote requests limited by concurrency limiter", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_worker_limiter_request_wait_duration_seconds", label_selectors=['instance=~"$worker_instance"'], is_optional_quantile=True, skip_default_instance=True, by_labels=["type"], ), legend_format="{{type}} - " + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Snapshot Request duration", heatmap_description="The time consumed to handle remote copr snapshot duration", graph_title="Snapshot Request duration", graph_description="The time consumed to handle remote copr snapshot duration", yaxis_format=UNITS.SECONDS, metric="tikv_worker_remote_cop_snapshot_duration_seconds", label_selectors=['instance=~"$worker_instance"'], skip_default_instance=True, ), ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Copr Request Process duration", heatmap_description="The time consumed to handle remote copr process duration", graph_title="Copr Request Process duration", graph_description="The time consumed to handle remote copr process duration", yaxis_format=UNITS.SECONDS, metric="tikv_worker_remote_cop_process_duration_seconds", label_selectors=['instance=~"$worker_instance"'], skip_default_instance=True, ), ) layout.row( heatmap_panel_graph_panel_histogram_quantile_pairs( heatmap_title="Copr Request duration", heatmap_description="The time consumed to handle remote coprocessor read requests", graph_title="Copr Request duration", graph_description="The time consumed to handle remote coprocessor read requests", yaxis_format=UNITS.SECONDS, metric="tikv_worker_remote_cop_request_duration_seconds", label_selectors=['instance=~"$worker_instance"'], skip_default_instance=True, ), ) layout.row( [ graph_panel( title="Remote Copr Request Operations", description="The QPS of completed remote coprocessor requests, including successful and failed outcomes, by top-level request type", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_remote_cop_requests_total", label_selectors=[ 'instance=~"$worker_instance"', 'outcome=~"success|failure"', ], by_labels=["request_type"], skip_default_instance=True, ), legend_format="{{request_type}}", additional_groupby=True, ), ], ), graph_panel( title="Remote Copr Response Size", description="The size of successful remote coprocessor responses", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_remote_cop_dag_response_size", label_selectors=['instance=~"$worker_instance"'], by_labels=[], skip_default_instance=True, ), legend_format="dag", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_worker_remote_analyze_response_size", label_selectors=['instance=~"$worker_instance"'], by_labels=[], skip_default_instance=True, ), legend_format="analyze", additional_groupby=True, ), target( expr=expr_sum_rate( "tikv_worker_remote_checksum_response_size", label_selectors=['instance=~"$worker_instance"'], by_labels=[], skip_default_instance=True, ), legend_format="checksum", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Remote Copr Scan Bytes", description="Processed and total bytes scanned by completed remote coprocessor requests", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_remote_cop_scan_bytes_total", label_selectors=['instance=~"$worker_instance"'], by_labels=["kind"], skip_default_instance=True, ), legend_format="{{kind}}", additional_groupby=True, ), ], ), graph_panel( title="Remote Copr Scan Keys", description="Processed and total keys scanned by completed remote coprocessor requests, split by request kind", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_remote_cop_scan_keys_total", label_selectors=['instance=~"$worker_instance"'], by_labels=["request_kind", "kind"], skip_default_instance=True, ), legend_format="{{request_kind}}-{{kind}}", additional_groupby=True, ), ], ), graph_panel( title="Memory Limiter Usage", description="Number of bytes are currently being used by requests limited by memory limiter", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_worker_memory_limiter_current_used", label_selectors=['instance=~"$worker_instance"'], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="TiKV/Cop Worker Duration (P99)", description="P99 duration of remote coprocessor prefetch", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_worker_remote_cop_prefetch_duration_seconds", label_selectors=['instance=~"$worker_instance"'], skip_default_instance=True, ), legend_format="prefetch", additional_groupby=True, ), ], ), graph_panel( title="Cop Worker IA Prefetch Cache Hit", description="P99 cache hit percentage of remote coprocessor prefetch", yaxes=yaxes(left_format=UNITS.PERCENT_FORMAT), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_worker_remote_cop_prefetch_cache_hit_percent", label_selectors=copr_worker_prefetch_selectors, by_labels=[], skip_default_instance=True, ), legend_format="prefetch cache hit", additional_groupby=True, ), ], ), ] ) return layout.row_panel def CloudWorkerNativeBR() -> RowPanel: layout = Layout(title="Cloud Worker - Native BR") worker_instance_selector = r'instance=~"${worker_instance:regex}"' layout.row( [ graph_panel( title="Native BR Duration", description=None, yaxes=yaxes(left_format=UNITS.SECONDS, log_base=1), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_worker_native_br_duration_seconds", label_selectors=[worker_instance_selector], is_optional_quantile=True, skip_default_instance=True, ), legend_format="total-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_worker_native_br_duration_seconds", label_selectors=[worker_instance_selector], by_labels=["type"], is_optional_quantile=True, skip_default_instance=True, ), legend_format="{{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_avg( "tikv_worker_native_br_duration_seconds", label_selectors=[worker_instance_selector], by_labels=["type"], skip_default_instance=True, ), legend_format="{{type}}-avg", additional_groupby=True, ), ], ), graph_panel( title="Native BR RPC Operations", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_worker_native_br_counter", label_selectors=[worker_instance_selector], by_labels=["type"], skip_default_instance=True, ), additional_groupby=True, ) ], ), ] ) layout.row( [ graph_panel( title="Native BR Backup Operations", description="Number of successful and failed backup operations", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "native_br_backup_success", label_selectors=[worker_instance_selector], skip_default_instance=True, ), legend_format="success-{{instance}}", additional_groupby=True, ), target( expr=expr_sum_rate( "native_br_backup_error", label_selectors=[worker_instance_selector], skip_default_instance=True, ), legend_format="error-{{instance}}", additional_groupby=True, ), ], ), graph_panel( title="Native BR Restore Errors", description="Number of errors during restoration by type", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "native_br_restore_error", label_selectors=[worker_instance_selector], by_labels=["type"], skip_default_instance=True, ), additional_groupby=True, ), target( expr=expr_sum_rate( "native_br_restore_rfengine_wal_epoch_overwritten_error", label_selectors=[worker_instance_selector], skip_default_instance=True, ), legend_format="wal_epoch_overwritten-{{instance}}", additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Native BR Restored Snapshot Bytes", description="Total bytes of restored snapshots", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "native_br_restored_snapshot_bytes", label_selectors=[worker_instance_selector], skip_default_instance=True, ), additional_groupby=True, ), ], ), heatmap_panel( title="Native BR Restore Snapshot Request Duration", description="Duration of restore snapshot of a shard", metric="native_br_restore_snapshot_request_secs_bucket", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), ), ] ) layout.row( [ graph_panel( title="Restore keyspace bytes", description="Pending and restored keyspace data sizes", yaxes=yaxes( left_format=UNITS.BYTES_IEC, right_format=UNITS.BYTES_SEC_IEC, ), targets=[ target( expr=expr_sum( "native_br_restore_pending_data_size", label_selectors=[worker_instance_selector], by_labels=[], skip_default_instance=True, ), legend_format="Pending data size", ), target( expr=expr_sum_rate( "native_br_restored_data_size", label_selectors=[worker_instance_selector], by_labels=[], skip_default_instance=True, ), legend_format="Restored data size rate", ), target( expr=expr_sum_rate( "native_br_restored_kv_size", label_selectors=[worker_instance_selector], by_labels=[], skip_default_instance=True, ), legend_format="Restored kv size rate", ), target( expr=expr_sum( "native_br_restored_data_size", label_selectors=[worker_instance_selector], by_labels=[], skip_default_instance=True, ), legend_format="Restored data size (total)", hide=True, ), target( expr=expr_sum( "native_br_restored_kv_size", label_selectors=[worker_instance_selector], by_labels=[], skip_default_instance=True, ), legend_format="Restored kv size (total)", hide=True, ), ], series_overrides=[ series_override( alias="/.*rate/", yaxis=2, ), ], ), ] ) return layout.row_panel def CloudWorkerSchemaManager() -> RowPanel: layout = Layout(title="Cloud Worker - Schema Manager") layout.row( [ graph_panel( title="Schema manager loops", description="Schema manager sync loops and errors per minute", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_increase( "tikv_worker_schema_manager_sync_loop_count", label_selectors=['instance=~"$worker_instance"'], range_selector="1m", by_labels=["instance"], skip_default_instance=True, ), legend_format="loops - {{instance}}", ), target( expr=expr_sum_increase( "tikv_worker_schema_manager_sync_loop_error_count", label_selectors=['instance=~"$worker_instance"'], range_selector="1m", by_labels=["type", "instance"], skip_default_instance=True, ), legend_format="error ({{type}}) - {{instance}}", ), ], ), ] ) return layout.row_panel def LocalFileGC() -> RowPanel: layout = Layout(title="Local File GC") layout.row( [ graph_panel( title="On Disk Files", description="The number of files on disk", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "rfstore_local_file_on_disk", ), ), ], ), graph_panel( title="Alive Files", description="The number of alive files used by kvengine", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "rfstore_local_file_alive", ), ), ], ), ] ) layout.row( [ graph_panel( title="GC Files", description="The number of files garbage collected", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "rfstore_local_file_gc_total", ), ), ], ), graph_panel( title="Skip GC Files", description="The number of files skipped by GC", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "rfstore_local_file_skip_gc", ), ), ], ), ] ) layout.row( [ graph_panel( title="Pending GC Files", description="The number of files pending for garbage collection", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum( "rfstore_local_file_pending_gc", ), ), ], ), graph_panel( title="GC Errors", description="The number of local file GC errors", yaxes=yaxes(left_format=UNITS.NONE_FORMAT), targets=[ target( expr=expr_sum_rate( "rfstore_local_file_error_total", by_labels=["type"], ), legend_format="{{type}}", additional_groupby=True, ), ], ), ] ) return layout.row_panel def PessimisticLocking() -> RowPanel: layout = Layout(title="Pessimistic Locking") layout.row( thread_cpu_panel_pair( title="Lock Manager Thread CPU", label_selector='name=~"(waiter_manager.*|deadlock_detect.*)"', by_labels=["instance", "name"], ) ) layout.row( [ graph_panel( title="Lock Manager Handled tasks", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_lock_manager_task_counter", by_labels=["type"], ), additional_groupby=True, ) ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Waiter lifetime duration", description="", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="tikv_lock_manager_waiter_lifetime_duration", hide_count=True, ), graph_panel( title="Lock Waiting Queue", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum_aggr_over_time( "tikv_lock_manager_wait_table_status", "max", "30s", by_labels=["type"], ), additional_groupby=True, ), target( expr=expr_sum_aggr_over_time( "tikv_lock_wait_queue_entries_gauge_vec", "max", "30s", by_labels=["type"], ), additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Deadlock detect duration", description="", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="tikv_lock_manager_detect_duration", hide_count=True, ), graph_panel( title="Detect error", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_lock_manager_error_counter", by_labels=["type"] ), additional_groupby=True, ) ], ), ] ) layout.row( [ graph_panel( title="Deadlock detector leader", targets=[ target( expr=expr_sum_aggr_over_time( "tikv_lock_manager_detector_leader_heartbeat", "max", "30s", ), ) ], ), graph_panel( title="Total pessimistic locks memory size", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_simple("tikv_pessimistic_lock_memory_size"), legend_format="{{instance}}", ) ], ), ] ) layout.row( [ graph_panel( title="In-memory pessimistic locking result", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_in_memory_pessimistic_locking", by_labels=["result"] ), additional_groupby=True, ) ], ), graph_panel( title="Pessimistic lock activities", description="The number of active keys and waiters.", targets=[ target( expr=expr_sum( "tikv_lock_wait_queue_entries_gauge_vec", by_labels=["type"] ), additional_groupby=True, ) ], ), ] ) layout.row( [ heatmap_panel( title="Shared lock holder count", description=( "Distribution of holder counts observed when writing non-empty " "shared-lock wrappers. Each observation represents one wrapper " "write, not the current live wrapper population." ), y_axis=yaxis(format=UNITS.SHORT, decimals=0), metric="tikv_shared_lock_holder_count_bucket", ), graph_panel( title="Shared lock upgrade conflicts", description="The rate of shared-lock upgrade conflicts by reason.", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_lock_upgrade_conflict_total", by_labels=["reason"] ), additional_groupby=True, ) ], ), ] ) layout.row( [ heatmap_panel( title="Lengths of lock wait queues when transaction enqueues", description="The length includes the entering transaction itself", y_axis=yaxis(format=UNITS.SHORT, decimals=1), metric="tikv_lock_wait_queue_length_bucket", ), graph_panel_histogram_quantiles( title="In-memory scan lock read duration", description="The duration scan in-memory pessimistic locks with read lock", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=2), metric="tikv_storage_mvcc_scan_lock_read_duration_seconds", by_labels=["type"], hide_count=True, hide_avg=True, ), ] ) return layout.row_panel def TxnFile() -> RowPanel: layout = Layout(title="Txn File") layout.row( [ graph_panel( title="Scheduler reject prewrite", description="total number of prewrite commands rejected by the lock limit", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), fill=1, targets=[ target( expr=expr_sum_rate( "tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total", by_labels=[], ), legend_format="total", ), target( expr=expr_sum_rate( "tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total", by_labels=["instance"], ), legend_format="{{instance}}", hide=True, ), ], ), graph_panel( title="Denied split", description="total number of denied splits due to locks or regions", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), fill=1, targets=[ target( expr=expr_sum_rate( "tikv_rfstore_txn_file_split_denied_locks_total", by_labels=["phase"], ), legend_format="locks - {{phase}}", ), target( expr=expr_sum_rate( "tikv_rfstore_txn_file_split_denied_regions_total", by_labels=["phase"], ), legend_format="regions - {{phase}}", ), ], ), ] ) layout.row( [ graph_panel( title="Resolve locks", description="total number of active resolve locks by result", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), fill=1, targets=[ target( expr=expr_sum_rate( "tikv_rfstore_txn_file_active_resolve_locks_total", by_labels=["result"], ), ), ], ), graph_panel( title="Resolve locks duration", description="duration of active resolve lock tasks", yaxes=yaxes(left_format=UNITS.SECONDS), fill=1, line_width=2, targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_rfstore_txn_file_active_resolve_task_duration_seconds", by_labels=[], ), legend_format="99%", ), target( expr=expr_histogram_quantile( 0.5, "tikv_rfstore_txn_file_active_resolve_task_duration_seconds", by_labels=[], ), legend_format="50%", hide=True, ), ], ), ] ) layout.row( [ graph_panel( title="Resolve locks tasks", description="active resolve lock running tasks, schedule results, and concurrency limit", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), fill=1, targets=[ target( expr=expr_sum( "tikv_rfstore_txn_file_active_resolve_running_tasks", by_labels=[], ), legend_format="running tasks", ), target( expr=expr_sum_rate( "tikv_rfstore_txn_file_active_resolve_schedule_total", by_labels=["result"], ), legend_format="schedule - {{result}}", ), target( expr=expr_avg( "tikv_rfstore_txn_file_active_resolve_concurrency_limit", by_labels=[], ), legend_format="concurrency limit", ), ], ), ] ) return layout.row_panel def PointInTimeRestore() -> RowPanel: layout = Layout(title="Point In Time Restore") layout.row( thread_cpu_panel_pair( title="CPU Usage", description=None, label_selector='name=~"sst_.*"', ) ) layout.row( [ graph_panel( title="P99 RPC Duration", description=None, yaxes=yaxes(left_format=UNITS.SECONDS, log_base=1), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_import_rpc_duration", label_selectors=[ 'request="apply"', ], is_optional_quantile=True, ), legend_format="total-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_import_apply_duration", label_selectors=[ 'type=~"queue|exec_download"', ], by_labels=["le", "type"], is_optional_quantile=True, ), legend_format="(DL){{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_import_engine_request", by_labels=["le", "type"], is_optional_quantile=True, ), legend_format="(AP){{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), ], ), ] ) layout.row( [ graph_panel( title="Import RPC Ops", description="", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_import_rpc_duration_count", label_selectors=[ 'request="apply"', ], by_labels=["instance", "request"], ), ), target( expr=expr_sum_rate( "tikv_import_rpc_duration_count", label_selectors=[ 'request!="switch_mode"', ], by_labels=["request"], ), legend_format="total-{{request}}", ), ], ), graph_panel( title="Import RPC Count", targets=[ target( expr=expr_simple( "tikv_import_rpc_count", label_selectors=[ 'type="apply"', ], ), legend_format="{{type}}-{{instance}}", ), ], ), graph_panel( title="Cache Events", description=None, yaxes=yaxes(left_format=UNITS.COUNTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_import_apply_cache_event", label_selectors=[], by_labels=["type", "instance"], ), ), ], ), ] ) layout.row( [ heatmap_panel( title="Overall RPC Duration", description=None, y_axis=yaxis(format=UNITS.SECONDS, log_base=1, decimals=1), metric="tikv_import_rpc_duration_bucket", label_selectors=[ 'request="apply"', ], ), heatmap_panel( title="Read File into Memory Duration", description=None, y_axis=yaxis(format=UNITS.SECONDS, log_base=1, decimals=1), metric="tikv_import_apply_duration_bucket", label_selectors=[ 'type="exec_download"', ], ), ] ) layout.row( [ heatmap_panel( title="Queuing Time", description=None, y_axis=yaxis(format=UNITS.SECONDS, log_base=1, decimals=1), metric="tikv_import_engine_request_bucket", label_selectors=[ 'type="queuing"', ], ), graph_panel( title="Apply Request Throughput", description=None, yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_import_apply_bytes_sum", ), ), ], ), ] ) layout.row( [ heatmap_panel( title="Downloaded File Size", description=None, y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), metric="tikv_import_download_bytes_bucket", ), heatmap_panel( title="Apply Batch Size", description=None, y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), metric="tikv_import_apply_bytes_bucket", ), ] ) layout.row( [ heatmap_panel( title="Blocked by Concurrency Time", description=None, y_axis=yaxis(format=UNITS.SECONDS, log_base=1, decimals=1), metric="tikv_import_engine_request_bucket", label_selectors=[ 'type="get_permit"', ], ), graph_panel( title="Apply Request Speed", description=None, yaxes=yaxes( left_format=UNITS.OPS_PER_SEC, log_base=1, ), targets=[ target( expr=expr_sum_rate( "tikv_import_applier_event", label_selectors=[ 'type="begin_req"', ], by_labels=["instance", "type"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Cached File in Memory", description=None, yaxes=yaxes(left_format=UNITS.BYTES_IEC, log_base=1), targets=[ target( expr=expr_sum("tikv_import_apply_cached_bytes"), ), ], ), graph_panel( title="Engine Requests Unfinished", description=None, yaxes=yaxes( left_format=UNITS.SHORT, log_base=1, ), targets=[ target( expr=expr_sum_rate( "tikv_import_applier_event", label_selectors=[ 'type!="begin_req"', ], by_labels=["instance", "type"], ), ), ], ), ] ) layout.row( [ heatmap_panel( title="Apply Time", description=None, y_axis=yaxis(format=UNITS.SECONDS, log_base=1, decimals=1), metric="tikv_import_engine_request_bucket", label_selectors=[ 'type="apply"', ], ), graph_panel( title="Raft Store Memory Usage", description="", yaxes=yaxes(left_format=UNITS.BYTES_IEC, log_base=1), targets=[ target( expr=expr_sum( "tikv_server_mem_trace_sum", label_selectors=[ 'name=~"raftstore-.*"', ], ), ), ], ), ] ) return layout.row_panel def ResolvedTS() -> RowPanel: layout = Layout(title="Resolved TS") layout.row( thread_cpu_panel_pair( title="Resolved TS Worker CPU", description="The CPU utilization of resolved ts worker", label_selector='name=~"resolved_ts.*"', ) ) layout.row( thread_cpu_panel_pair( title="Advance ts Worker CPU", description="The CPU utilization of advance ts worker", label_selector='name=~"advance_ts.*"', ) ) layout.row( thread_cpu_panel_pair( title="Scan lock Worker CPU", description="The CPU utilization of scan lock worker", label_selector='name=~"inc_scan.*"', ) ) layout.row( [ graph_panel( title="Max gap of resolved-ts", description="The gap between resolved ts (the maximum candidate of safe-ts) and current time.", yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=expr_sum( "tikv_resolved_ts_min_resolved_ts_gap_millis", ), ) ], ), graph_panel( title="Min Resolved TS Region", description="The region that has minimal resolved ts", targets=[ target( expr=expr_sum( "tikv_resolved_ts_min_resolved_ts_region", ), ) ], ), ] ) layout.row( [ graph_panel( title="Max gap of follower safe-ts", description="The gap between now() and the minimal (non-zero) safe ts for followers", yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=expr_sum( "tikv_resolved_ts_min_follower_safe_ts_gap_millis", ), ) ], ), graph_panel( title="Min Safe TS Follower Region", description="The region id of the follower that has minimal safe ts", targets=[ target( expr=expr_sum( "tikv_resolved_ts_min_follower_safe_ts_region", ), ) ], ), ] ) layout.row( [ graph_panel( title="Max gap of resolved-ts in region leaders", description="The gap between resolved ts of leaders and current time", yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=expr_sum( "tikv_resolved_ts_min_leader_resolved_ts_gap_millis", ), ) ], ), graph_panel( title="Min Leader Resolved TS Region", description="The region that its leader has minimal resolved ts.", targets=[ target( expr=expr_sum( "tikv_resolved_ts_min_leader_resolved_ts_region", ), ) ], ), ] ) layout.row( [ heatmap_panel( title="Check leader duration", description="The time consumed when handle a check leader request", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_resolved_ts_check_leader_duration_seconds_bucket", ), graph_panel( title="CheckLeader request region count " + OPTIONAL_QUANTILE_INPUT, description="Bucketed histogram of region count in a check leader request", targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_check_leader_request_item_count", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-" + OPTIONAL_QUANTILE_INPUT, ) ], ), ] ) layout.row( [ graph_panel( title="CheckLeader request size" + OPTIONAL_QUANTILE_INPUT, description="Bucketed histogram of the check leader request size", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_check_leader_request_size_bytes", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "tikv_check_leader_request_item_count", by_labels=["instance"], is_optional_quantile=True, ), legend_format="{{instance}}-check-num-" + OPTIONAL_QUANTILE_INPUT, ), ], ), graph_panel( title="Fail advance ts count", description="The count of fail to advance resolved-ts", targets=[ target( expr=expr_sum_delta( "tikv_resolved_ts_fail_advance_count", by_labels=["instance", "reason"], ), ), target( expr=expr_sum_delta( "tikv_raftstore_check_stale_peer", by_labels=["instance"], ), legend_format="{{instance}}-stale-peer", ), ], ), ] ) layout.row( [ graph_panel( title="Lock heap size", description="Total bytes in memory of resolved-ts observe regions's lock heap", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_avg( "tikv_resolved_ts_lock_heap_bytes", ), ) ], ), heatmap_panel( title="Initial scan backoff duration", description="The backoff duration before starting initial scan", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket", ), ] ) layout.row( [ graph_panel( title="Observe region status", description="The status of resolved-ts observe regions", targets=[ target( expr=expr_sum( "tikv_resolved_ts_region_resolve_status", by_labels=["type"], ), additional_groupby=True, ) ], ), graph_panel( title="Pending command size", description="Total bytes of pending commands in the channel", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_avg( "tikv_resolved_ts_channel_penging_cmd_bytes_total", ), ) ], ), ] ) return layout.row_panel def Memory() -> RowPanel: layout = Layout(title="Memory") layout.row( [ graph_panel( title="Allocator Stats", description=None, yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_allocator_stats", by_labels=["instance", "type"] ) ) ], ), graph_panel( title="Send Allocated(+) / Release Received(-) Bytes Rate", description=None, yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_operator( expr_sum_rate( "tikv_allocator_thread_allocation", label_selectors=['type="alloc"'], by_labels=["thread_name"], ), "-", expr_sum_rate( "tikv_allocator_thread_allocation", label_selectors=['type="dealloc"'], by_labels=["thread_name"], ), ), legend_format="{{thread_name}}", additional_groupby=True, ) ], ), ] ) layout.row( [ graph_panel( title="Allocated Bytes Rate per Thread", description=None, yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_allocator_thread_allocation", label_selectors=['type="alloc"'], by_labels=["thread_name"], ), additional_groupby=True, ) ], ), graph_panel( title="Released Bytes Rate per Thread", description=None, yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum_rate( "tikv_allocator_thread_allocation", label_selectors=['type="dealloc"'], by_labels=["thread_name"], ), additional_groupby=True, ) ], ), ] ) layout.row( [ graph_panel( title="Mapped Allocation per Thread", description=None, yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum( "tikv_allocator_thread_stats", label_selectors=['type="mapped"'], by_labels=["thread_name"], ), additional_groupby=True, ) ], ), graph_panel( title="Arena Count", description=None, targets=[ target( expr=expr_sum( "tikv_allocator_arena_count", by_labels=["instance"], ), ) ], ), ] ) layout.row( [ graph_panel( title="Memory Profile Auto Dump Actions", description="Auto dump action rates grouped by component and action", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_memory_profile_auto_dump_total", label_selectors=['job=~".*(tikv|copr|schema-manager).*"'], by_labels=["component", "action"], ), legend_format=r"{{component}}-{{action}}", ) ], ), ] ) return layout.row_panel def BackupImport() -> RowPanel: layout = Layout(title="Backup & Import") layout.row( thread_cpu_panel_pair( title="Backup CPU Utilization", label_selector='name=~"(b.*k.*w.*k.*|backup_io)"', by_labels=["instance", "name"], ) ) layout.row( [ graph_panel( title="Backup Thread Count", targets=[ target( expr=expr_sum( "tikv_backup_thread_pool_size", ), ), ], ), graph_panel( title="Backup Errors", description="", targets=[ target( expr=expr_sum_delta( "tikv_backup_error_counter", by_labels=["instance", "error"], ), ), ], ), ] ) layout.row( [ heatmap_panel( title="Backup Write CF SST Size", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), metric="tikv_backup_range_size_bytes_bucket", label_selectors=['cf="write"'], ), heatmap_panel( title="Backup Default CF SST Size", y_axis=yaxis(format=UNITS.BYTES_IEC, decimals=1), metric="tikv_backup_range_size_bytes_bucket", label_selectors=['cf="default"'], ), graph_panel( title="Backup SST Generation Throughput", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_backup_range_size_bytes_sum", by_labels=[], # override default by instance. ), legend_format="total", ), target( expr=expr_sum_rate( "tikv_backup_range_size_bytes_sum", by_labels=["instance", "cf"], ), ), ], ), ] ) layout.row( [ heatmap_panel( title="Backup Scan SST Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_backup_range_duration_seconds_bucket", label_selectors=['type="snapshot"'], ), heatmap_panel( title="Backup Scan SST Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_backup_range_duration_seconds_bucket", label_selectors=['type="scan"'], ), heatmap_panel( title="Backup Save SST Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_backup_range_duration_seconds_bucket", label_selectors=['type=~"save.*"'], ), graph_panel( title="Backup SST Duration", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.999, "tikv_backup_range_duration_seconds", by_labels=["type"], ), legend_format="{{type}}-99.9%", additional_groupby=True, ), target( expr=expr_histogram_quantile( 0.99, "tikv_backup_range_duration_seconds", by_labels=["type"], is_optional_quantile=True, ), legend_format="{{type}}-" + OPTIONAL_QUANTILE_INPUT, additional_groupby=True, ), target( expr=expr_operator( expr_sum( "tikv_backup_range_duration_seconds_sum", by_labels=["type"], ), "/", expr_sum( "tikv_backup_range_duration_seconds_count", by_labels=["type"], ), ), legend_format="{{type}}-avg", additional_groupby=True, ), ], ), ] ) layout.row( [ heatmap_panel( title="External Storage Create Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_external_storage_create_seconds_bucket", ), graph_panel_histogram_quantiles( title="External Storage Create Duration", description="", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_external_storage_create_seconds", hide_avg=True, hide_count=True, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Checksum Request Duration", description="", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_coprocessor_request_duration_seconds", label_selectors=['req=~"analyze.*|checksum.*"'], by_labels=["req"], hide_avg=True, hide_count=True, ), graph_panel( title="IO Utilization", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_sum_rate( "node_disk_io_time_seconds_total", by_labels=["instance", "device"], ), ), ], ), ] ) layout.row( thread_cpu_panel_pair( title="Import CPU Utilization", label_selector='name=~"sst_.*"', by_labels=["instance", "tid"], additional_groupby=True, ) ) layout.row( [ graph_panel( title="Import Thread Count", targets=[ target( expr=expr_count_rate( "tikv_thread_cpu_seconds_total", label_selectors=['name=~"sst_.*"'], by_labels=["instance"], ), legend_format="{{instance}}", ), ], ), graph_panel( title="Import Errors", targets=[ target( expr=expr_sum_delta( "tikv_import_error_counter", by_labels=["type", "error", "instance"], ), ), ], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Import RPC Duration", description="", yaxes=yaxes(left_format=UNITS.SECONDS), metric="tikv_import_rpc_duration", by_labels=["request"], hide_count=True, ), graph_panel( title="Import RPC Ops", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_import_rpc_duration_count", label_selectors=['request!="switch_mode"'], by_labels=["request"], ), additional_groupby=True, ), ], ), graph_panel( title="Import RPC Count", targets=[ target( expr=expr_simple( "tikv_import_rpc_count", ), legend_format="{{type}}-{{instance}}", ), ], ), ] ) layout.row( [ heatmap_panel( title="Import Write/Download RPC Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_rpc_duration_bucket", label_selectors=['request=~"download|write"'], ), heatmap_panel( title="Import Wait Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_download_duration_bucket", label_selectors=['type="queue"'], ), heatmap_panel( title="Import Read SST Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_download_duration_bucket", label_selectors=['type="read"'], ), heatmap_panel( title="Import Rewrite SST Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_download_duration_bucket", label_selectors=['type="rewrite"'], ), ] ) layout.row( [ heatmap_panel( title="Import Ingest RPC Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_rpc_duration_bucket", label_selectors=['request=~"ingest"'], ), heatmap_panel( title="Import Ingest SST Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_ingest_duration_bucket", label_selectors=['type=~"ingest"'], ), heatmap_panel( title="Import Ingest SST Bytes", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_import_ingest_byte_bucket", ), graph_panel( title="Import Download SST Throughput", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_import_download_bytes_sum", ), ), target( expr=expr_sum_rate( "tikv_import_download_bytes_sum", by_labels=[], ), legend_format="total", ), ], ), ] ) layout.row( [ graph_panel( title="Import Local Write keys", targets=[ target( expr=expr_sum_delta( "tikv_import_local_write_keys", by_labels=["type", "instance"], ), ), ], ), graph_panel( title="Import Local Write bytes", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_import_local_write_bytes", by_labels=["type", "instance"], ), ), ], ), ] ) layout.row( [ graph_panel( title="TTL Expired", targets=[ target( expr=expr_sum( "tikv_backup_raw_expired_count", ), ), target( expr=expr_sum( "tikv_backup_raw_expired_count", by_labels=[], ), legend_format="sum", ), ], ), graph_panel( title="cloud request", description="", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum_rate( "tikv_cloud_request_duration_seconds_count", by_labels=["cloud", "req"], ), ), ], ), ] ) return layout.row_panel def Encryption() -> RowPanel: layout = Layout(title="Encryption") layout.row( [ graph_panel( title="Encrypted region count", description="Number of regions with encryption enabled", targets=[ target( expr=expr_sum( "tikv_encryption_region_count", ), legend_format="{{instance}}", ), ], ), graph_panel( title="Encrypted keyspace count", description="Number of keyspaces with encryption enabled", targets=[ target( expr=expr_sum( "tikv_encryption_keyspace_active_count", ), legend_format="{{instance}}", ), ], ), graph_panel( title="Encryption errors", targets=[ target( expr=expr_sum_rate( "tikv_encryption_key_unavailable_count", by_labels=["instance", "reason"], ), legend_format=r"key-unavailable-{{instance}}-{{reason}}", ), target( expr=expr_sum( "tikv_encryption_keyspace_cmek_health_check_failure_count", ), legend_format=r"health-check-failed-{{instance}}", ), target( expr=expr_sum( "tikv_rfstore_encryption_apply_paused_peers", ), legend_format=r"cmek-apply-paused-{{instance}}", ), ], thresholds=[GraphThreshold(value=0.0)], ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Raft log encrypt/decrypt duration", description="Time spent encrypting or decrypting raft log entries in the write path.", metric="rfstore_raft_entry_crypt_duration_seconds", yaxes=yaxes(left_format=UNITS.SECONDS), by_labels=["type"], hide_p9999=True, hide_avg=True, ), graph_panel_histogram_quantiles( title="KV engine SST encrypt/decrypt duration", description="Time spent encrypting or decrypting SST data in the KV engine.", metric="kv_engine_sst_crypt_duration_seconds", yaxes=yaxes(left_format=UNITS.SECONDS), by_labels=["type"], hide_p9999=True, hide_avg=True, ), ] ) layout.row( [ graph_panel_histogram_quantiles( title="Master Key Generate/Decrypt duration (calls KMS)", description="Master key generating or decrypting duration, including KMS round trips.", metric="tikv_master_key_ops_duration_seconds", yaxes=yaxes(left_format=UNITS.SECONDS), by_labels=["type"], hide_p9999=True, hide_avg=True, ), graph_panel_histogram_quantiles( title="Encryption Key Read Duration", description="Encryption Key read duration.", metric="tikv_encryption_key_get_duration", yaxes=yaxes(left_format=UNITS.SECONDS), hide_p9999=True, hide_avg=True, ), ] ) layout.row( [ graph_panel( title="Encrypted key switch rate", description="Rate of TiKV switching to a previous data key to decrypt older data.", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_encryption_key_switch_count", ), legend_format="{{instance}}", ), ], ), ] ) return layout.row_panel def BackupLog() -> RowPanel: layout = Layout(title="Backup Log") layout.row( [ stat_panel( title="Endpoint Status", targets=[ target( expr=expr_simple("tikv_log_backup_enabled"), legend_format="{{ instance }}", ), ], mappings=[ StatValueMappings( StatValueMappingItem("Disabled", "0", "red"), StatValueMappingItem("Enabled", "1", "green"), ), ], ), stat_panel( title="Task Status", targets=[ target( expr=expr_min("tikv_log_backup_task_status"), ), ], mappings=[ StatValueMappings( StatValueMappingItem("Running", "0", "green"), StatValueMappingItem("Paused", "1", "yellow"), StatValueMappingItem("Error", "2", "red"), ), ], ), stat_panel( title="Advancer Owner", text_mode="name", targets=[ target( expr="tidb_log_backup_advancer_owner > 0", legend_format="{{ instance }}", ), ], ), stat_panel( title="Average Flush Size", description="The average flush size of last 30mins.", format=UNITS.BYTES_IEC, targets=[ target( expr=expr_operator( expr_sum_increase( "tikv_log_backup_flush_file_size_sum", range_selector="30m", ), "/", expr_sum_increase( "tikv_log_backup_flush_duration_sec_count", label_selectors=['stage=~"save_files"'], range_selector="30m", ), ), legend_format="{{ instance }}", ), ], ), ] ) layout.row( [ stat_panel( title="Flushed Files (Last 30m) Per Host", description="The current total flushed file number of this run.", decimals=0, targets=[ target( expr=expr_sum_delta( "tikv_log_backup_flush_file_size_count", range_selector="30m", ).extra("> 0"), ), ], ), stat_panel( title="Flush Times (Last 30m)", description="This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", decimals=0, targets=[ target( expr=expr_sum_delta( "tikv_log_backup_flush_duration_sec_count", range_selector="30m", label_selectors=['stage=~"save_files"'], ), ), ], ), stat_panel( title="Total Flushed Size (Last 30m)", description="This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", format=UNITS.BYTES_IEC, targets=[ target( expr=expr_sum_delta( "tikv_log_backup_flush_file_size_sum", range_selector="30m", ), ), ], ), stat_panel( title="Flush Files (Last 30m)", description="This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", decimals=0, targets=[ target( expr=expr_sum_delta( "tikv_log_backup_flush_file_size_count", range_selector="30m", ), ), ], ), ] ) layout.row( [ stacked_thread_cpu_panel( title="CPU Usage", description="The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", label_selector='name=~"(backup_stream|log-backup-scan(-[0-9]+)?)"', by_labels=["instance", "name"], ), graph_panel( title="Handle Event Rate", description="", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_log_backup_handle_kv_batch_sum", ), ) ], ), ] ) layout.row( [ graph_panel( title="Initial Scan Generate Event Throughput", description="The data rate of initial scanning emitting events.", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_log_backup_incremental_scan_bytes_sum", ), ) ], ), graph_panel( title="Abnormal Checkpoint TS Lag", description=None, yaxes=yaxes(left_format=UNITS.MILLI_SECONDS), targets=[ target( expr=expr_operator( "time() * 1000", "-", expr_max( "tidb_log_backup_last_checkpoint", by_labels=["task"] ).extra("/ 262144 > 0"), ), legend_format="{{ task }}", ), ], ), ] ) layout.row( [ graph_panel( title="Memory Of Events", description="The estimated memory usage by the streaming backup module.", yaxes=yaxes(left_format=UNITS.BYTES_IEC), targets=[ target( expr=expr_sum("tikv_log_backup_heap_memory"), ) ], ), graph_panel( title="Observed Region Count", description="", targets=[ target( expr=expr_sum("tikv_log_backup_observed_region"), ), target( expr=expr_sum( "tikv_log_backup_observed_region", ), legend_format="{{instance}}-total", ), ], ), ] ) layout.row( [ graph_panel( title="Errors", description="The errors met when backing up.\n**They are retryable, don't worry.**", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_delta( "tikv_log_backup_errors", range_selector="1m", by_labels=["type", "instance"], ), ), ], ), graph_panel( title="Fatal Errors", description="The errors met when backing up.", yaxes=yaxes(left_format=UNITS.OPS_PER_MIN), targets=[ target( expr=expr_sum_delta( "tikv_log_backup_fatal_errors", range_selector="1m", by_labels=["type", "instance"], ), ), ], ), graph_panel( title="Checkpoint TS of Tasks", description=None, yaxes=yaxes(left_format=UNITS.DATE_TIME_ISO_TODAY), null_point_mode=NULL_AS_NULL, targets=[ target( expr=expr_max( "tidb_log_backup_last_checkpoint", by_labels=["task"] ).extra("/ 262144 > 0"), ), target(expr="time() * 1000", legend_format="Current Time"), ], series_overrides=[ series_override( alias="Current Time", fill=0, dashes=True, ), ], ), ] ) layout.row( [ heatmap_panel( title="Flush Duration", description="The duration of flushing a batch of file.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_flush_duration_sec_bucket", label_selectors=['stage=~"save_files"'], ), heatmap_panel( title="Initial scanning duration", description="The duration of scanning the initial data from local DB and transform them into apply events.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_initial_scan_duration_sec_bucket", ), heatmap_panel( title="Convert Raft Event duration", description="The duration of converting a raft request into a apply event.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_event_handle_duration_sec_bucket", label_selectors=['stage=~"to_stream_event"'], ), heatmap_panel( title="Wait for Lock Duration", description="The duration of waiting the mutex of the controller.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_event_handle_duration_sec_bucket", label_selectors=['stage=~"get_router_lock"'], ), ] ) layout.row( [ heatmap_panel( title="Command Batch Size", description="The number of KV-modify of each raft command observed.", y_axis=yaxis(format=UNITS.SHORT, decimals=1), metric="tikv_log_backup_handle_kv_batch_bucket", ), heatmap_panel( title="Save to Temp File Duration", description="The total cost of saving an event into temporary file.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_event_handle_duration_sec_bucket", label_selectors=['stage=~"save_to_temp_file"'], ), heatmap_panel( title="Write to Temp File Duration", description="The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_on_event_duration_seconds_bucket", label_selectors=['stage="write_to_tempfile"'], ), heatmap_panel( title="System Write Call Duration", description="The duration of collecting metadata and call the UNIX system call *write* for each event.", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tikv_log_backup_on_event_duration_seconds_bucket", label_selectors=['stage="syscall_write"'], ), ] ) layout.row( [ graph_panel( title="Internal Message Type", description="The internal message type count.", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC, log_base=2), targets=[ target( expr=expr_sum_rate( "tikv_log_backup_internal_actor_acting_duration_sec_count", by_labels=["message"], ), ) ], ), graph_panel( title="Internal Message Handling Duration " + OPTIONAL_QUANTILE_INPUT, description="The internal handling message duration.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_log_backup_internal_actor_acting_duration_sec", by_labels=["message"], is_optional_quantile=True, ), legend_format="{{message}}-" + OPTIONAL_QUANTILE_INPUT, ) ], ), graph_panel( title="Internal Message Handling Duration (P90)", description="The internal handling message duration.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.9, "tikv_log_backup_internal_actor_acting_duration_sec", by_labels=["message"], ), legend_format="{{message}}", ) ], ), ] ) layout.row( [ graph_panel( title="Initial Scan RocksDB Throughput", description="The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), targets=[ target( expr=expr_sum_rate( "tikv_log_backup_initial_scan_operations", label_selectors=['op=~"read_bytes"'], by_labels=["cf"], ), ) ], ), graph_panel( title="Initial Scan RocksDB Operation", description="Misc statistics of RocksDB during initial scanning.", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_log_backup_initial_scan_operations", label_selectors=['op!~"read_bytes"'], by_labels=["cf", "op"], ).extra("> 0"), ) ], ), ] ) layout.row( [ graph_panel( title="Initial Scanning Trigger Reason", description="The reason of triggering initial scanning.", targets=[ target( expr=expr_sum_rate( "tikv_log_backup_initial_scan_reason", by_labels=["reason"], ), ) ], ), graph_panel( title="Initial Scanning Task Status", description="The task status of initial scanning.", targets=[ target( expr=expr_sum( "tikv_log_backup_pending_initial_scan", by_labels=["stage"], ), ) ], ), graph_panel( title="Region Checkpoint Key Putting", description="", yaxes=yaxes(left_format=UNITS.COUNTS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_log_backup_metadata_key_operation", by_labels=["type"], ), ) ], ), ] ) layout.row( [ heatmap_panel( title="Request Checkpoint Batch Size", metric="tidb_log_backup_advancer_batch_size_bucket", label_selectors=['type="checkpoint"'], ), heatmap_panel( title="Tick Duration", y_axis=yaxis(format=UNITS.SECONDS, decimals=1), metric="tidb_log_backup_advancer_tick_duration_sec_bucket", label_selectors=['step="tick"'], ), ] ) layout.row( [ graph_panel( title="Region Checkpoint Failure Reason", description="The reason of advancer failed to be advanced.", targets=[ target( expr=expr_sum_rate( "tidb_log_backup_region_request_failure", label_selectors=['reason!="retryable-scan-region"'], by_labels=["reason"], ), ), ], ), graph_panel( title="Request Result", description="The result of getting region checkpoints.", targets=[ target( expr=expr_sum_rate( "tidb_log_backup_region_request", by_labels=["result"], ), ), ], ), ] ) layout.row( [ graph_panel( title="Tick Duration " + OPTIONAL_QUANTILE_INPUT, description="The internal handling message duration.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tidb_log_backup_advancer_tick_duration_sec", by_labels=["step"], is_optional_quantile=True, ), legend_format="{{step}}-" + OPTIONAL_QUANTILE_INPUT, ) ], ), graph_panel( title="Tick Duration (P90)", description="The internal handling message duration.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.9, "tidb_log_backup_advancer_tick_duration_sec", by_labels=["step"], ), legend_format="{{ step }}", ) ], ), ] ) layout.row( [ graph_panel( title="Get Region Operation Count", description="The frequent of getting region level checkpoint.", targets=[ target( expr=expr_sum_rate( "tidb_log_backup_advancer_tick_duration_sec_count", label_selectors=['step="get-regions-in-range"'], by_labels=["step", "instance"], ), ) ], ), graph_panel( title="Try Advance Trigger Time", description="The variant of checkpoint group.", targets=[ target( expr=expr_sum_rate( "tidb_log_backup_advancer_tick_duration_sec_count", label_selectors=['step="try-advance"'], by_labels=["step", "instance"], ), ) ], ), ] ) return layout.row_panel def SlowNodeStatistics() -> RowPanel: layout = Layout(title="Slow Node Statistics") layout.row( [ graph_panel( title="Inspected duration per server", description="The duration that recorded by inspecting messages.", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "store_inspect_disk_duration_seconds", by_labels=["instance", "type"], is_optional_quantile=True, ), legend_format="disk-{{instance}}-{{type}}-" + OPTIONAL_QUANTILE_INPUT, ), target( expr=expr_histogram_quantile( 0.99, "store_inspect_network_duration_seconds", by_labels=["instance", "target"], is_optional_quantile=True, ), legend_format="network-{{instance}}-{{target}}-" + OPTIONAL_QUANTILE_INPUT, ), ], ), graph_panel( title="Store Slow Score", description="The slow score of stores", targets=[ target( expr=expr_sum( "store_slow_score", by_labels=["instance", "type"], ), ), ], ), ] ) return layout.row_panel def StatusServer() -> RowPanel: layout = Layout(title="Status Server") layout.row( [ graph_panel_histogram_quantiles( title="Status API Request Duration", description="The 99 quantile durtion of status server API requests", metric="tikv_status_server_request_duration_seconds", yaxes=yaxes(left_format=UNITS.SECONDS), by_labels=["path"], hide_p9999=True, hide_count=True, hide_avg=True, ), graph_panel( title="Status API Request (op/s)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_status_server_request_duration_seconds_count", by_labels=["path"], ), additional_groupby=True, ), ], ), ] ) return layout.row_panel def ResourceControl() -> RowPanel: layout = Layout(title="Resource Control") layout.row( [ graph_panel( title="Background Task Total Wait Duration", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_sum_rate( "tikv_resource_control_background_task_wait_duration", by_labels=["instance", "resource_group"], ), ), ], ), graph_panel( title="Priority Quota Limit", description="The memory usage of the resource control module.", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_sum( "tikv_resource_control_priority_quota_limit", by_labels=["instance", "priority"], ), ), ], ), ] ) # CPU Throttle Metrics layout.row( [ graph_panel( title="CPU Throttle Token Allocation Rate", description="Rate of CPU throttle token allocations by result", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_cpu_throttle_allocations_total", by_labels=["instance", "result"], ), legend_format="{{instance}}-{{ result }}", ), target( expr=expr_sum_rate( "tikv_cpu_throttle_low_priority_dag_allocations_total", by_labels=["instance", "result"], ), legend_format="{{instance}}-low-pri-{{ result }}", ), ], ), graph_panel( title="Global CPU Usage Ratio", description="Global CPU usage ratio (0.0 - 1.0)", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_avg( "tikv_cpu_usage_monitor_global_ratio", by_labels=["instance"], ), legend_format="{{ instance }}", ), ], ), ] ) layout.row( [ graph_panel( title="Per-Keyspace CPU Usage Ratio", description="Per-keyspace CPU usage ratio (0.0 - 1.0)", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_avg( "tikv_cpu_usage_monitor_keyspace_ratio", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), graph_panel( title="CPU Throttle Refill Rate Adjustments", description="Rate of refill rate adjustments (dynamic adjustment)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_cpu_throttle_refill_rate_adjustments_total", by_labels=["instance", "level", "direction"], ), legend_format="{{instance}}-{{ level }}-{{ direction }}", ), ], ), ] ) layout.row( [ graph_panel( title="CPU Throttle Burst Count", description="Number of burst allocations per keyspace", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_cpu_throttle_burst_total", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), graph_panel( title="CPU Throttle Burst Tokens Used", description="Rate of CPU tokens used in burst mode (microseconds)", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_sum_rate( "tikv_cpu_throttle_burst_tokens_used_total", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), ] ) layout.row( [ graph_panel( title="CPU Throttle Fair Allocations", description="Rate of fair allocation triggers when global CPU > threshold", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_cpu_throttle_fair_allocations_total", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), graph_panel( title="CPU Token Allocation Retries", description="Rate of retries during CPU token allocation", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_cpu_token_allocation_retries_total", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), ] ) # Tenant Max CPU Cap (uniform per-tenant cap mode) layout.row( [ graph_panel( title="Tenant CPU Cap Ratio", description="Current uniform per-tenant CPU cap ratio (0 = disabled)", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_max( "tikv_cpu_throttle_tenant_cap_ratio", by_labels=["instance"], ), legend_format="{{ instance }}", ), ], ), graph_panel( title="Tenant CPU Cap Quota", description=( "Computed uniform per-tenant CPU cap quota " "(microseconds per refill interval; 0 = disabled)" ), yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_max( "tikv_cpu_throttle_tenant_cap_quota_us", by_labels=["instance"], ), legend_format="{{ instance }}", ), ], ), graph_panel( title="Tenant CPU Cap Exhausted", description=( "Rate of allocations rejected because the per-tenant CPU " "cap was exhausted with no burst available" ), yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "tikv_cpu_throttle_allocations_total", label_selectors=['result="tenant_cap_exhausted"'], by_labels=["instance"], ), legend_format="{{ instance }}", ), ], ), ] ) # CPU Token Wait Duration Metrics layout.row( [ graph_panel( title="CPU Token Wait Duration (Avg)", description="Average wait duration for CPU token allocation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_cpu_token_wait_duration_seconds", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id", "result"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}-{{ result }}", ), target( expr=expr_histogram_avg( "tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds", by_labels=["instance", "result"], ), legend_format="low-pri-{{ instance }}-{{ result }}", ), ], ), graph_panel( title="CPU Token Wait Duration (P99)", description="P99 wait duration for CPU token allocation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_token_wait_duration_seconds", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id", "result"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}-{{ result }}", ), target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds", by_labels=["instance", "result"], ), legend_format="low-pri-{{ instance }}-{{ result }}", ), ], ), ] ) layout.row( [ graph_panel( title="Low-Priority DAG Wait Duration", description="Success-path low-priority DAG total wait duration", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_cpu_throttle_low_priority_dag_wait_duration_seconds", label_selectors=['result="success"'], by_labels=["instance"], ), legend_format="total-avg-{{ instance }}", ), target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_throttle_low_priority_dag_wait_duration_seconds", label_selectors=['result="success"'], by_labels=["instance"], ), legend_format="total-p99-{{ instance }}", ), ], ), ] ) layout.row( [ graph_panel( title="Global Token Bucket Availability Ratio", description="Available tokens / capacity for global bucket", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_avg( "tikv_cpu_throttle_global_bucket_available_us", by_labels=["instance"], ), "/", expr_avg( "tikv_cpu_throttle_global_bucket_capacity_us", by_labels=["instance"], ), ), legend_format="{{ instance }}", ), target( expr=expr_operator( expr_avg( "tikv_cpu_throttle_low_priority_dag_bucket_available_us", by_labels=["instance"], ), "/", expr_avg( "tikv_cpu_throttle_low_priority_dag_bucket_capacity_us", by_labels=["instance"], ), ), legend_format="low-pri-{{ instance }}", ), ], ), graph_panel( title="Keyspace Token Bucket Availability Ratio", description="Available tokens / capacity for keyspace buckets", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_operator( expr_avg( "tikv_cpu_throttle_keyspace_bucket_available_us", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), "/", expr_avg( "tikv_cpu_throttle_keyspace_bucket_capacity_us", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), ] ) layout.row( [ graph_panel( title="Global Token Bucket Refill Rate", description="Current global refill rate (us per interval)", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_avg( "tikv_cpu_throttle_global_refill_rate_us_per_interval", by_labels=["instance"], ), legend_format="{{ instance }}", ), target( expr=expr_avg( "tikv_cpu_throttle_low_priority_dag_refill_rate_us_per_interval", by_labels=["instance"], ), legend_format="low-pri-{{ instance }}", ), ], ), graph_panel( title="Keyspace Token Bucket Refill Rate", description="Current keyspace refill rate (us per interval)", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_avg( "tikv_cpu_throttle_keyspace_refill_rate_us_per_interval", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), ] ) layout.row( [ graph_panel( title="Runtime Token Wait Duration (Avg)", description="Average wait duration for runtime token allocation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_cpu_token_runtime_wait_duration_seconds", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id", "result"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}-{{ result }}", ), ], ), graph_panel( title="Runtime Token Wait Duration (P99)", description="P99 wait duration for runtime token allocation", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_token_runtime_wait_duration_seconds", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id", "result"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}-{{ result }}", ), ], ), ] ) layout.row( [ graph_panel( title="Throttled Request CPU Time (Avg)", description="Average CPU time per throttled request", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_avg( "tikv_cpu_throttle_request_cpu_time_seconds", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), target( expr=expr_histogram_avg( "tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds", by_labels=["instance"], ), legend_format="low-pri-{{ instance }}", ), ], ), graph_panel( title="Throttled Request CPU Time (P99)", description="P99 CPU time per throttled request", yaxes=yaxes(left_format=UNITS.SECONDS), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_throttle_request_cpu_time_seconds", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds", by_labels=["instance"], ), legend_format="low-pri-{{ instance }}", ), ], ), ] ) layout.row( [ graph_panel( title="Actual/Estimated CPU Ratio (Avg)", description="Average ratio of actual CPU time to estimated", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_histogram_avg( "tikv_cpu_throttle_request_actual_to_estimated_ratio", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), target( expr=expr_histogram_avg( "tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio", by_labels=["instance"], ), legend_format="low-pri-{{ instance }}", ), ], ), graph_panel( title="Actual/Estimated CPU Ratio (P99)", description="P99 ratio of actual CPU time to estimated", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_throttle_request_actual_to_estimated_ratio", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), target( expr=expr_histogram_quantile( 0.99, "tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio", by_labels=["instance"], ), legend_format="low-pri-{{ instance }}", ), ], ), ] ) layout.row( [ graph_panel( title="Service Limit Keyspace Value", description="Current keyspace service limit value from PD (RU/s)", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_avg( "tikv_service_limit_keyspace_value", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), graph_panel( title="Service Limit Keyspace Quota", description="CPU quota derived from PD service limit (us per interval)", yaxes=yaxes(left_format=UNITS.MICRO_SECONDS), targets=[ target( expr=expr_avg( "tikv_service_limit_keyspace_quota_us", label_selectors=['keyspace_id=~"$keyspace_id"'], by_labels=["instance", "keyspace_id"], ), legend_format="{{ instance }}-KS{{ keyspace_id }}", ), ], ), ] ) return layout.row_panel def ResourceControlServerless() -> RowPanel: layout = Layout(title="Resource Control (Serverless)") layout.row( [ graph_panel( title="Request wait count", description="count of different kinds of request wait", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), fill=1, targets=[ target( expr=expr_sum_rate( "tikv_request_wait_duration_seconds_count", by_labels=["instance", "type"], ), legend_format="{{instance}}-{{type}}", ), ], ), graph_panel( title="Keyspace read MBps", description="total bytes of read in each active keyspace", yaxes=yaxes(left_format=UNITS.BYTES_SEC_IEC), fill=0, targets=[ target( expr=expr_sum_rate( "tikv_active_keyspace_read_bytes", by_labels=["instance", "keyspace_id"], ), legend_format="{{instance}}-{{keyspace_id}}-read", ), ], ), ] ) layout.row( [ graph_panel( title="Average request wait duration", description="wait time of request", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=10), fill=1, targets=[ target( expr=expr_histogram_avg( "tikv_request_wait_duration_seconds", by_labels=["instance", "type"], ), legend_format="{{instance}}-{{type}}", ), ], ), graph_panel( title="99% request wait duration", description="wait time of request", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=10), fill=1, targets=[ target( expr=expr_histogram_quantile( 0.99, "tikv_request_wait_duration_seconds", by_labels=["instance", "type"], ), legend_format="{{instance}}-{{type}}", ), ], ), ] ) layout.row( [ graph_panel( title="99.99% request wait duration", description="wait time of request", yaxes=yaxes(left_format=UNITS.SECONDS, log_base=10), fill=1, targets=[ target( expr=expr_histogram_quantile( 0.9999, "tikv_request_wait_duration_seconds", by_labels=["instance", "type"], ), legend_format="{{instance}}-{{type}}", ), ], ), ] ) return layout.row_panel def ColdRegion() -> RowPanel: layout = Layout(title="Cold Region") layout.row( [ graph_panel( title="Cold Regions", description="Current local regions whose peer is in the Cold state", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum("rfstore_cold_regions_count"), legend_format="{{instance}}", ), ], ), graph_panel( title="Cold Region State Transition Rate", description="Rate of cold-region state transitions by destination state", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "rfstore_cold_region_state_transitions_total", by_labels=["instance", "state"], ), legend_format="{{instance}}-{{state}}", ), ], ), ] ) layout.row( [ graph_panel( title="Pending Cold Leader Probes", description="Cold-region leader probes waiting for their timeout", yaxes=yaxes(left_format=UNITS.SHORT), targets=[ target( expr=expr_sum("rfstore_cold_leader_probe_pending_count"), legend_format="{{instance}}", ), ], ), graph_panel( title="Cold Campaign Rate", description="Rate of cold-region campaign attempts by reason", yaxes=yaxes(left_format=UNITS.OPS_PER_SEC), targets=[ target( expr=expr_sum_rate( "rfstore_cold_campaign_events_total", by_labels=["instance", "reason"], ), legend_format="{{instance}}-{{reason}}", ), ], ), ] ) return layout.row_panel def PressureStall() -> RowPanel: layout = Layout(title="Pressure Stall (PSI)") def psi_panel(resource: str, title: str) -> Panel: targets = [] for level in ["some", "full"]: for window in ["avg10", "avg60"]: base = expr_simple( "tikv_pressure_stall_average", label_selectors=[ f'resource="{resource}"', f'level="{level}"', f'window="{window}"', ], ) targets.append( target( expr=expr_operator(base, "/", 100), legend_format=f"{level}-{window}-{{{{instance}}}}", ) ) return graph_panel( title=title, description="PSI averages (percent) for some/full stalls over 10s/60s windows.", yaxes=yaxes(left_format=UNITS.PERCENT_UNIT), targets=targets, line_width=1, fill=0, fill_gradient=0, stack=False, ) layout.row( [ psi_panel("cpu", "CPU PSI"), psi_panel("io", "IO PSI"), psi_panel("memory", "Memory PSI"), ] ) return layout.row_panel #### Metrics Definition End #### dashboard = Dashboard( title="Test-Cluster-TiKV-Details", uid="RDVQiEzZz", timezone="browser", refresh="1m", inputs=[DATASOURCE_INPUT], editable=True, templating=Templates(), panels=[ # Overview Duration(), Cluster(), Errors(), Server(), # Entrance of Write and Read gRPC(), Storage(), LocalReader(), # CPU and IO ThreadCPU(), PressureStall(), IOBreakdown(), # Raftstore RaftWaterfall(), RaftIO(), RaftPropose(), RaftProcess(), RaftMessage(), RaftAdmin(), RaftLog(), # Engine RaftEngine(), KvEngine(), KvEngineCompaction(), IA(), # Not supported. # InMemoryEngine(), CloudWorkerService(), CloudWorkerNativeBR(), CloudWorkerSchemaManager(), LocalFileGC(), # Scheduler and Read Pools FlowControl(), Scheduler(), SchedulerWorkerPool(), Yatp(), SchedulerCommands(), CoprocessorOverview(), CoprocessorDetail(), UnifiedReadPool(), # Transaction GC(), PessimisticLocking(), TxnFile(), # Background Tasks Task(), PD(), SlowNodeStatistics(), Snapshot(), # Tools ResolvedTS(), PointInTimeRestore(), BackupImport(), BackupLog(), # Advanced Debugging for CPU and Memory Threads(), Memory(), # Infrequently Used ResourceControl(), ResourceControlServerless(), StatusServer(), Encryption(), TTL(), ], # Set 14 or larger to support shared crosshair or shared tooltip. # See https://github.com/grafana/grafana/blob/v10.2.2/public/app/features/dashboard/state/DashboardMigrator.ts#L443-L445 schemaVersion=14, graphTooltip=GRAPH_TOOLTIP_MODE_SHARED_CROSSHAIR, ).auto_panel_ids()