local grafana = import 'grafonnet-lib/grafonnet/grafana.libsonnet';
local dashboard = grafana.dashboard;
local row = grafana.row;
local graphPanel = grafana.graphPanel;
local prometheus = grafana.prometheus;
local template = grafana.template;

local myNameFlag = 'DS_TEST-CLUSTER';
local myDS = '${' + myNameFlag + '}';

// A new dashboard
local newDash = dashboard.new(
  title='Test-Cluster-TiCI-Worker',
  editable=true,
  graphTooltip='shared_crosshair',
  refresh='30s',
  time_from='now-1h',
)
.addInput(
  name=myNameFlag,
  label='test-cluster',
  type='datasource',
  pluginId='prometheus',
  pluginName='Prometheus',
)
.addTemplate(
  template.new(
    datasource=myDS,
    hide= 2,
    label='K8s-cluster',
    name='k8s_cluster',
    query='label_values(pd_cluster_status, k8s_cluster)',
    refresh='time',
    sort=1,
  )
)
.addTemplate(
  // Default template for tidb-cloud
  template.new(
    allValues=null,
    current=null,
    datasource=myDS,
    hide='all',
    includeAll=false,
    label='tidb_cluster',
    multi=false,
    name='tidb_cluster',
    query='label_values(pd_cluster_status{k8s_cluster="$k8s_cluster"}, tidb_cluster)',
    refresh='time',
    regex='',
    sort=1,
    tagValuesQuery='',
  )
).addTemplate(
  // Default template for tidb-cloud
  template.new(
    allValues=null,
    current=null,
    datasource=myDS,
    hide='',
    includeAll=true,
    label='Instance',
    multi=false,
    name='instance',
    query='query_result(max by (instance) (up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", component="tici-worker"} or up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", job="tici-worker"} or up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"tici-worker-.+"} or up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", job="tici"}))',
    refresh='time',
    regex='/.*instance="([^"]+)".*/',
    sort=1,
    tagValuesQuery='',
  )
);

local workerMetricMatcher =
  'k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"';
local workerBatchQueueMatcher =
  workerMetricMatcher + ', component=~"metadata_store|frag_batch_uploader"';

// Server row and its panels
local serverRow = row.new(collapse=true, title='Server');
local uptimeP = graphPanel.new(
  title='Uptime',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Uptime since the last restart.',
)
.addTarget(
  prometheus.target(
    'time() - process_start_time_seconds{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local cpuP = graphPanel.new(
  title='CPU Usage',
  datasource=myDS,
  legend_rightSide=true,
  description='',
  format='percentunit',
)
.addTarget(
  prometheus.target(
    'rate(process_cpu_seconds_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])',
    legendFormat='{{instance}}',
  )
);
/*
// TODO: Add cpu quota
.addTarget(
  prometheus.target(
    'tidb_server_maxprocs{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='quota-{{instance}}',
  )
);
*/

local memP = graphPanel.new(
  title='Memory Usage',
  datasource=myDS,
  legend_rightSide=true,
  description='',
  format='bytes',
)
.addTarget(
  prometheus.target(
    'process_resident_memory_bytes{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local processMemoryStatusPanel = graphPanel.new(
  title='Process Memory Status',
  datasource=myDS,
  legend_rightSide=true,
  fill=0,
  description=
    'Process memory fields collected from /proc/self/status. Values are converted from kB to bytes.\n\n' +
    'VmRSS: current resident set size in physical memory.\n\n' +
    'RssAnon: resident anonymous memory such as heap and stack.\n\n' +
    'RssFile: resident file-backed memory such as mapped files and shared libraries.\n\n' +
    'RssShmem: resident shared memory including tmpfs and shared anonymous mappings.\n\n' +
    'VmHWM: peak resident set size since process start.',
  format='bytes',
)
.addTarget(
  prometheus.target(
    'tici_process_memory_status_bytes{' + workerMetricMatcher + '}',
    legendFormat='{{type}}-{{instance}}',
  )
);

/*
// TODO: Add memory quota
.addTarget(
  prometheus.target(
    'tidb_server_memory_quota_bytes{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='quota-{{instance}}',
  )
);*/

// CDC row
local cdcRow = row.new(collapse=true, title='CDC');
local cdcEventOPSPanel = graphPanel.new(
  title='CDC Event OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='CDC event ops.',
)
.addTarget(
  prometheus.target(
    'sum by (type) (rate(tici_writer_cdc_event_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}',
  )
);

local cdcEventTotalPanel = graphPanel.new(
  title='CDC Event Total Count',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Total CDC event count accumulated over time.',
)
.addTarget(
  prometheus.target(
    'sum by (type) (tici_writer_cdc_event_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{type}}',
  )
);

local cdcOperationPanel = graphPanel.new(
  title='CDC Operations',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type) (rate(tici_writer_cdc_operation_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}',
  )
);

local cdcOperationLatencyPanel = graphPanel.new(
  title='CDC Operation Latency',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_writer_cdc_operation_latency_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_writer_cdc_operation_latency_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_writer_cdc_operation_latency_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

// Fragment row
local fragRow = row.new(collapse=true, title='Frag');

local fragOperationPanel = graphPanel.new(
  title='Fragment Operations',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type) (rate(tici_writer_frag_operation_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}',
  )
);

local fragOperationLatencyPanel = graphPanel.new(
  title='Fragment Operation Latency',
  datasource=myDS,
  legend_rightSide=true,
  format='ms',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_writer_frag_operation_latency_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_writer_frag_operation_latency_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_writer_frag_operation_latency_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local workerCompactionOPS = graphPanel.new(
  title='Compaction OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (stage, result) (rate(tici_worker_compaction_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{stage}}-{{result}}',
  )
);

local workerCompactionDuration = graphPanel.new(
  title='Compaction Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_worker_compaction_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, stage))',
    legendFormat='{{stage}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_worker_compaction_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage) / sum(rate(tici_worker_compaction_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage)',
    legendFormat='{{stage}}-avg',
  )
);

local workerCompactionInflight = graphPanel.new(
  title='Compaction Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'sum by (stage) (tici_worker_compaction_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{stage}}',
  )
);

local workerCompactionFragmentCount = graphPanel.new(
  title='Compaction Fragment Count',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_worker_compaction_fragment_count_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, name))',
    legendFormat='{{name}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_worker_compaction_fragment_count_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (name) / sum(rate(tici_worker_compaction_fragment_count_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (name)',
    legendFormat='{{name}}-avg',
  )
);

local workerCompactionFragmentSize = graphPanel.new(
  title='Compaction Fragment Size',
  datasource=myDS,
  legend_rightSide=true,
  format='bytes',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_worker_compaction_fragment_size_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, name))',
    legendFormat='{{name}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_worker_compaction_fragment_size_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (name) / sum(rate(tici_worker_compaction_fragment_size_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (name)',
    legendFormat='{{name}}-avg',
  )
);

local fragUploadRow = row.new(collapse=true, title='Frag Upload');

local fragUploadGroupCountPanel = graphPanel.new(
  title='Upload Group Count',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Upload groups produced by one fragment upload request.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_writer_frag_upload_group_count_bucket{' + workerMetricMatcher + '}[1m])) by (le))',
    legendFormat='p99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_writer_frag_upload_group_count_sum{' + workerMetricMatcher + '}[1m])) / sum(rate(tici_writer_frag_upload_group_count_count{' + workerMetricMatcher + '}[1m]))',
    legendFormat='avg',
  )
);

local fragUploadGroupFileCountPanel = graphPanel.new(
  title='Upload Group File Count',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Files packed into each fragment upload group.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_writer_frag_upload_group_file_count_bucket{' + workerMetricMatcher + '}[1m])) by (le))',
    legendFormat='p99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_writer_frag_upload_group_file_count_sum{' + workerMetricMatcher + '}[1m])) / sum(rate(tici_writer_frag_upload_group_file_count_count{' + workerMetricMatcher + '}[1m]))',
    legendFormat='avg',
  )
);

local fragUploadGroupBytesPanel = graphPanel.new(
  title='Upload Group Bytes',
  datasource=myDS,
  legend_rightSide=true,
  format='bytes',
  description='Bytes packed into each fragment upload group.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_writer_frag_upload_group_bytes_bucket{' + workerMetricMatcher + '}[1m])) by (le))',
    legendFormat='p99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_writer_frag_upload_group_bytes_sum{' + workerMetricMatcher + '}[1m])) / sum(rate(tici_writer_frag_upload_group_bytes_count{' + workerMetricMatcher + '}[1m]))',
    legendFormat='avg',
  )
);

local fragUploadMergedFileCountPanel = graphPanel.new(
  title='Merged File OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Files avoided by merging uploads into grouped objects.',
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_writer_frag_upload_merged_file_count{' + workerMetricMatcher + '}[1m]))',
    legendFormat='merged',
  )
);

// Shard writer manager row and its panels
local shardWriterManagerRow = row.new(collapse=true, title='Shard Writer Manager');

local shardWriterManagerOperationOPS = graphPanel.new(
  title='Shard Writer Manager OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_shard_writer_manager_operation_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local shardWriterManagerOperationDuration = graphPanel.new(
  title='Shard Writer Manager Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_shard_writer_manager_operation_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_shard_writer_manager_operation_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_shard_writer_manager_operation_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local shardWriterManagerOperationInflight = graphPanel.new(
  title='Shard Writer Manager Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'sum by (type) (tici_shard_writer_manager_operation_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{type}}',
  )
);

// Import row and its panels
local importRow = row.new(collapse=true, title='Import Into');

local importInflightTasksPanel = graphPanel.new(
  title='Import Tasks Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Import tasks in flight.',
)
.addTarget(
  prometheus.target(
    'tici_worker_import_into_inflight_tasks{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local importRowsProcessedTotalPanel = graphPanel.new(
  title='Import Rows',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Total rows processed by import into.',
)
.addTarget(
  prometheus.target(
    'tici_worker_import_into_rows_processed_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local importBytesProcessedTotalPanel = graphPanel.new(
  title='Import Bytes',
  datasource=myDS,
  legend_rightSide=true,
  format='bytes',
  description='Total bytes processed by import into.',
)
.addTarget(
  prometheus.target(
    'tici_worker_import_into_bytes_processed_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

// GRPC Row
local grpcRow = row.new(collapse=true, title='GRPC');
local grpcRequestDuration = graphPanel.new(
  title='Grpc Request Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
  .addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_grpc_handle_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"}[1m])) by (le, type))',
    legendFormat='{{type}}-99%',
  )
);

local grpcRequestOPS = graphPanel.new(
  title='Grpc Request OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_grpc_handle_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local grpcRequestInflight = graphPanel.new(
  title='Grpc Request Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'sum by (service, type) (tici_grpc_handle_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"})',
    legendFormat='{{service}}-{{type}}',
  )
);

// Meta client row and its panels
local metaClientRow = row.new(collapse=true, title='Meta Client');

local metaClientGrpcDuration = graphPanel.new(
  title='Meta Client Grpc Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_meta_client_grpc_handle_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_meta_client_grpc_handle_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_meta_client_grpc_handle_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local metaClientGrpcOPS = graphPanel.new(
  title='Meta Client Grpc OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_meta_client_grpc_handle_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local metaClientGrpcInflight = graphPanel.new(
  title='Meta Client Grpc Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'sum by (type) (tici_meta_client_grpc_handle_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{type}}',
  )
);

local metaClientPoolFailedAttempts = graphPanel.new(
  title='Meta Client Pool Failed Attempts',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Failed attempts to get a meta client pool channel.',
)
.addTarget(
  prometheus.target(
    'sum by (instance) (rate(tici_meta_client_pool_failed_attempts{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{instance}}',
  )
);

local batchQueueRow = row.new(collapse=true, title='Batch Queue');

local batchQueueSubmitOPSPanel = graphPanel.new(
  title='Submit OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Batch queue submit rate by component and submit result.',
)
.addTarget(
  prometheus.target(
    'sum by (component, result) (rate(tici_batch_queue_submit_count{' + workerBatchQueueMatcher + '}[1m]))',
    legendFormat='{{component}}-{{result}}',
  )
);

local batchQueueRequestOPSPanel = graphPanel.new(
  title='Request OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Batch queue completion rate and merged-request rate.',
)
.addTarget(
  prometheus.target(
    'sum by (component, result) (rate(tici_batch_queue_request_count{' + workerBatchQueueMatcher + '}[1m]))',
    legendFormat='done-{{component}}-{{result}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (rate(tici_batch_queue_merged_request_count{' + workerBatchQueueMatcher + '}[1m]))',
    legendFormat='merged-{{component}}',
  )
);

local batchQueueQueueDurationPanel = graphPanel.new(
  title='Queue Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Wait time before enqueue and before execution.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_enqueue_block_duration_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='enqueue-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_enqueue_block_duration_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_enqueue_block_duration_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='enqueue-{{component}}-avg',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_queue_duration_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='queue-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_queue_duration_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_queue_duration_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='queue-{{component}}-avg',
  )
);

local batchQueueRequestDurationPanel = graphPanel.new(
  title='Request Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='End-to-end request latency from submit to response.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_request_duration_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_request_duration_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_request_duration_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='{{component}}-avg',
  )
);

local batchQueueBatchDurationPanel = graphPanel.new(
  title='Batch Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Batch collection and execution duration by component.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_collect_duration_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='collect-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_collect_duration_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_collect_duration_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='collect-{{component}}-avg',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_execute_duration_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='execute-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_execute_duration_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_execute_duration_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='execute-{{component}}-avg',
  )
);

local batchQueueBacklogPanel = graphPanel.new(
  title='Backlog',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Queued request and weight backlog by component.',
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_request_backlog{' + workerBatchQueueMatcher + '})',
    legendFormat='request-{{component}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_weight_backlog{' + workerBatchQueueMatcher + '})',
    legendFormat='weight-{{component}}',
  )
);

local batchQueueInflightPanel = graphPanel.new(
  title='Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Inflight submit calls, running batches, and running requests.',
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_submit_inflight{' + workerBatchQueueMatcher + '})',
    legendFormat='submit-{{component}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_batch_inflight{' + workerBatchQueueMatcher + '})',
    legendFormat='batch-{{component}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_request_inflight{' + workerBatchQueueMatcher + '})',
    legendFormat='request-{{component}}',
  )
);

local batchQueueBatchShapePanel = graphPanel.new(
  title='Batch Shape',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Batch size in requests and accumulated batch weight.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_size_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='size-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_size_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_size_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='size-{{component}}-avg',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_weight_bucket{' + workerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='weight-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_weight_sum{' + workerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_weight_count{' + workerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='weight-{{component}}-avg',
  )
);

local batchQueueFlushOPSPanel = graphPanel.new(
  title='Flush OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Batch flush trigger rate by component and reason.',
)
.addTarget(
  prometheus.target(
    'sum by (component, reason) (rate(tici_batch_queue_batch_flush_count{' + workerBatchQueueMatcher + '}[1m]))',
    legendFormat='{{component}}-{{reason}}',
  )
);

// Dedup row and its panels
local dedupRow = row.new(collapse=true, title='Dedup');

local dedupRunOPS = graphPanel.new(
  title='Dedup Run OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (result) (rate(tici_dedup_run_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{result}}',
  )
);

local dedupInProgress = graphPanel.new(
  title='Dedup In Progress',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'tici_dedup_in_progress{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local dedupStageLatency = graphPanel.new(
  title='Dedup Stage Latency',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_dedup_stage_latency_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, stage))',
    legendFormat='{{stage}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_dedup_stage_latency_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage) / sum(rate(tici_dedup_stage_latency_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage)',
    legendFormat='{{stage}}-avg',
  )
);

// Storage row and its panels
local storageRow = row.new(collapse=true, title='Storage');

local storageHandleDuration = graphPanel.new(
  title='Storage Handle Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Backend storage request duration by backend and operation type',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_storage_handle_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"}[1m])) by (le, backend, type))',
    legendFormat='{{backend}}/{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_storage_handle_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"}[1m])) by (backend, type) / sum(rate(tici_storage_handle_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"}[1m])) by (backend, type)',
    legendFormat='{{backend}}/{{type}}-avg',
  )
);

local storageHandleCount = graphPanel.new(
  title='Storage Handle Count',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Backend storage request rate by backend, operation type, and result',
)
.addTarget(
  prometheus.target(
    'sum by (backend, type, result) (rate(tici_storage_handle_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"}[1m]))',
    legendFormat='{{backend}}/{{type}}-{{result}}',
  )
);

local storageHandleInflight = graphPanel.new(
  title='Storage Handle Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Inflight backend storage requests by backend and operation type',
)
.addTarget(
  prometheus.target(
    'sum by (backend, type) (tici_storage_handle_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="worker"})',
    legendFormat='{{backend}}/{{type}}',
  )
);

// Merge together.
local panelW = 12;
local panelH = 7;
local rowW = 24;
local rowH = 1;

local rowPos = {x:0, y:0, w:rowW, h:rowH};
local fullPanelPos = {x:0, y:0, w:rowW, h:panelH};
local leftPanelPos = {x:0, y:0, w:panelW, h:panelH};
local rightPanelPos = {x:panelW, y:0, w:panelW, h:panelH};

newDash
.addPanel(
  serverRow
  .addPanel(uptimeP, gridPos=leftPanelPos)
  .addPanel(cpuP, gridPos=rightPanelPos)
  .addPanel(memP, gridPos=leftPanelPos)
  .addPanel(processMemoryStatusPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  cdcRow
  .addPanel(cdcEventOPSPanel, gridPos=leftPanelPos)
  .addPanel(cdcEventTotalPanel, gridPos=rightPanelPos)
  .addPanel(cdcOperationPanel, gridPos=leftPanelPos)
  .addPanel(cdcOperationLatencyPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  fragRow
  .addPanel(fragOperationPanel, gridPos=leftPanelPos)
  .addPanel(fragOperationLatencyPanel, gridPos=rightPanelPos)
  .addPanel(workerCompactionOPS, gridPos=leftPanelPos)
  .addPanel(workerCompactionDuration, gridPos=rightPanelPos)
  .addPanel(workerCompactionInflight, gridPos=leftPanelPos)
  .addPanel(workerCompactionFragmentCount, gridPos=rightPanelPos)
  .addPanel(workerCompactionFragmentSize, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  fragUploadRow
  .addPanel(fragUploadGroupCountPanel, gridPos=leftPanelPos)
  .addPanel(fragUploadGroupFileCountPanel, gridPos=rightPanelPos)
  .addPanel(fragUploadGroupBytesPanel, gridPos=leftPanelPos)
  .addPanel(fragUploadMergedFileCountPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  shardWriterManagerRow
  .addPanel(shardWriterManagerOperationOPS, gridPos=leftPanelPos)
  .addPanel(shardWriterManagerOperationDuration, gridPos=rightPanelPos)
  .addPanel(shardWriterManagerOperationInflight, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  importRow
  .addPanel(importInflightTasksPanel, gridPos=leftPanelPos)
  .addPanel(importRowsProcessedTotalPanel, gridPos=rightPanelPos)
  .addPanel(importBytesProcessedTotalPanel, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  grpcRow
  .addPanel(grpcRequestDuration, gridPos=leftPanelPos)
  .addPanel(grpcRequestOPS, gridPos=rightPanelPos)
  .addPanel(grpcRequestInflight, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  metaClientRow
  .addPanel(metaClientGrpcDuration, gridPos=leftPanelPos)
  .addPanel(metaClientGrpcOPS, gridPos=rightPanelPos)
  .addPanel(metaClientGrpcInflight, gridPos=leftPanelPos)
  .addPanel(metaClientPoolFailedAttempts, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  batchQueueRow
  .addPanel(batchQueueSubmitOPSPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueRequestOPSPanel, gridPos=rightPanelPos)
  .addPanel(batchQueueQueueDurationPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueRequestDurationPanel, gridPos=rightPanelPos)
  .addPanel(batchQueueBatchDurationPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueBacklogPanel, gridPos=rightPanelPos)
  .addPanel(batchQueueInflightPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueBatchShapePanel, gridPos=rightPanelPos)
  .addPanel(batchQueueFlushOPSPanel, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  dedupRow
  .addPanel(dedupRunOPS, gridPos=leftPanelPos)
  .addPanel(dedupInProgress, gridPos=rightPanelPos)
  .addPanel(dedupStageLatency, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  storageRow
  .addPanel(storageHandleDuration, gridPos=leftPanelPos)
  .addPanel(storageHandleCount, gridPos=rightPanelPos)
  .addPanel(storageHandleInflight, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)