local grafana = import 'grafonnet-lib/grafonnet/grafana.libsonnet';
local dashboard = grafana.dashboard;
local row = grafana.row;
local graphPanel = grafana.graphPanel;
local prometheus = grafana.prometheus;
local template = grafana.template;

local myNameFlag = 'DS_TEST-CLUSTER';
local myDS = '${' + myNameFlag + '}';

// A new dashboard
local newDash = dashboard.new(
  title='Test-Cluster-TiCI-Reader',
  editable=true,
  graphTooltip='shared_crosshair',
  refresh='30s',
  time_from='now-1h',
)
.addInput(
  name=myNameFlag,
  label='test-cluster',
  type='datasource',
  pluginId='prometheus',
  pluginName='Prometheus',
)
.addTemplate(
  template.new(
    datasource=myDS,
    hide= 2,
    label='K8s-cluster',
    name='k8s_cluster',
    query='label_values(pd_cluster_status, k8s_cluster)',
    refresh='time',
    sort=1,
  )
)
.addTemplate(
  // Default template for tidb-cloud
  template.new(
    allValues=null,
    current=null,
    datasource=myDS,
    hide='all',
    includeAll=false,
    label='tidb_cluster',
    multi=false,
    name='tidb_cluster',
    query='label_values(pd_cluster_status{k8s_cluster="$k8s_cluster"}, tidb_cluster)',
    refresh='time',
    regex='',
    sort=1,
    tagValuesQuery='',
  )
).addTemplate(
  // Default template for tidb-cloud
  template.new(
    allValues=null,
    current=null,
    datasource=myDS,
    hide='',
    includeAll=true,
    label='Instance',
    multi=false,
    name='instance',
    query='query_result(max by (instance) (up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", component=~".*tiflash"} or up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", job=~".*tiflash.*"} or up{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", job="tiflash"}))',
    refresh='time',
    regex='/.*instance="([^"]+)".*/',
    sort=1,
    tagValuesQuery='',
  )
);

local readerMetricMatcher =
  'k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"';
local readerBatchQueueMatcher =
  readerMetricMatcher + ', component="metadata_store"';

// Server row and its panels
local serverRow = row.new(collapse=true, title='Server');
local uptimeP = graphPanel.new(
  title='Uptime',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Uptime since the last restart.',
)
.addTarget(
  prometheus.target(
    'tiflash_system_asynchronous_metric_Uptime{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local cpuP = graphPanel.new(
  title='CPU Usage',
  datasource=myDS,
  legend_rightSide=true,
  description='',
  format='percentunit',
)
.addTarget(
  prometheus.target(
    'rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])',
    legendFormat='{{instance}}',
  )
);

local memP = graphPanel.new(
  title='Memory Usage',
  datasource=myDS,
  legend_rightSide=true,
  description='',
  format='bytes',
)
.addTarget(
  prometheus.target(
    'tiflash_proxy_process_resident_memory_bytes{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local processMemoryStatusPanel = graphPanel.new(
  title='Process Memory Status',
  datasource=myDS,
  legend_rightSide=true,
  fill=0,
  description=
    'Process memory fields collected from /proc/self/status. Values are converted from kB to bytes.\n\n' +
    'VmRSS: current resident set size in physical memory.\n\n' +
    'RssAnon: resident anonymous memory such as heap and stack.\n\n' +
    'RssFile: resident file-backed memory such as mapped files and shared libraries.\n\n' +
    'RssShmem: resident shared memory including tmpfs and shared anonymous mappings.\n\n' +
    'VmHWM: peak resident set size since process start.',
  format='bytes',
)
.addTarget(
  prometheus.target(
    'tici_process_memory_status_bytes{' + readerMetricMatcher + '}',
    legendFormat='{{type}}-{{instance}}',
  )
);

local shardRow = row.new(collapse=true, title='Shard');
local CountPanel = graphPanel.new(
  title='Count',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Shard count and frag count.',
)
.addTarget(
  prometheus.target(
    'tici_reader_shard_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='Shard Count-{{instance}}',
  )
)
.addTarget(
  prometheus.target(
    'tici_reader_frag_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='Frag Count-{{instance}}',
  )
);

local metadataScanSkippedPanel = graphPanel.new(
  title='Metadata Scan Skipped',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description=
    'Metadata rows skipped during best-effort local restore scans. ' +
    'reason=iterator means redb failed while iterating or reading a metadata item, indicating possible local metadata DB corruption or local disk/IO issues. ' +
    'SOP: check reader logs for "Failed to access metadata item", inspect disk/redb health, and treat repeated growth as urgent. ' +
    'reason=deserialize means the item was readable but its JSON metadata could not be decoded as the expected type/schema, often stale or corrupted local metadata. ' +
    'SOP: check warning logs for the skipped key/prefix and recent upgrade/rollback; usually not urgent if the count is low and stable, but investigate if it grows or restore misses shards/indexes/frags.',
)
.addTarget(
  prometheus.target(
    'sum by (instance, prefix, reason) (tici_metadata_scan_skipped_total{' + readerMetricMatcher + '})',
    legendFormat='{{instance}}-{{prefix}}-{{reason}}',
  )
);

local warmupRow = row.new(collapse=true, title='Warmup');
local warmupDurationPanel = graphPanel.new(
  title='Warmup Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Reader warmup stage duration by stage',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_warmup_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, stage))',
    legendFormat='{{stage}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_warmup_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage) / sum(rate(tici_reader_warmup_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage)',
    legendFormat='{{stage}}-avg',
  )
);

local heartbeatRow = row.new(collapse=true, title='Heartbeat');
local shardRefreshDurationPanel = graphPanel.new(
  title='Shard Refresh Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Reader shard refresh duration by refresh type.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_shard_refresh_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_shard_refresh_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_reader_shard_refresh_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local shardRefreshCountPanel = graphPanel.new(
  title='Shard Refresh Count',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Reader shard refresh rate by refresh type and result.',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_reader_shard_refresh_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local shardRefreshQueueDurationPanel = graphPanel.new(
  title='Shard Refresh Queue Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Queued time before reader shard refresh starts.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_shard_refresh_queue_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le))',
    legendFormat='p99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_shard_refresh_queue_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) / sum(rate(tici_reader_shard_refresh_queue_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='avg',
  )
);

local heartbeatQueueBacklogPanel = graphPanel.new(
  title='Heartbeat Queue Backlog',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Shard refresh tasks waiting in the reader heartbeat response queue.',
)
.addTarget(
  prometheus.target(
    'tici_reader_heartbeat_queue_backlog{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local shardRefreshInflightPanel = graphPanel.new(
  title='Shard Refresh Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Inflight reader shard refresh work.',
)
.addTarget(
  prometheus.target(
    'sum by (type) (tici_reader_shard_refresh_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{type}}',
  )
);

local shardRefreshEventCountPanel = graphPanel.new(
  title='Shard Refresh Event Count',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Queue lifecycle event rate for reader shard refresh.',
)
.addTarget(
  prometheus.target(
    'sum by (type) (rate(tici_reader_shard_refresh_event_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}',
  )
);

local queryRow = row.new(collapse=true, title='Query');
local queryLatencyPanel = graphPanel.new(
  title='Query Latency',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Query latency.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_query_latency_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.90, sum(rate(tici_reader_query_latency_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-90',
  )
);

local shardQueryDurationPanel = graphPanel.new(
  title='Shard Query Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Wall-clock duration of shard reader search streams from open to drop (includes caller idle time between next_batch calls).',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_shard_query_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_shard_query_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_reader_shard_query_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local shardQueryOpenDurationPanel = graphPanel.new(
  title='Shard Query Open Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Active processing time spent opening a shard reader search stream.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_shard_query_open_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_shard_query_open_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_reader_shard_query_open_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local shardQueryNextDurationPanel = graphPanel.new(
  title='Shard Query Next Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Total active processing time spent in next_batch across a shard reader search stream.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_shard_query_next_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_shard_query_next_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_reader_shard_query_next_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local shardQueryBatchCountPanel = graphPanel.new(
  title='Shard Query Batch Count',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Number of non-empty next_batch calls per shard reader search stream.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_shard_query_batch_count_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_shard_query_batch_count_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_reader_shard_query_batch_count_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local shardQueryCountPanel = graphPanel.new(
  title='Shard Query Count',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Shard reader query count.',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_reader_shard_query_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local shardQueryInflightPanel = graphPanel.new(
  title='Shard Query Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Inflight shard reader queries.',
)
.addTarget(
  prometheus.target(
    'sum by (type) (tici_reader_shard_query_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{type}}',
  )
);

local docCacheRow = row.new(collapse=true, title='Doc Cache');
local docCacheAccessPanel = graphPanel.new(
  title='Doc Cache Access QPS',
  datasource=myDS,
  legend_alignAsTable=true,
  legend_rightSide=true,
  legend_sideWidth=180,
  format='ops',
  description='Doc cache access rate by access result and resident hit rate on the right Y axis.',
)
.resetYaxes()
.addYaxis(format='ops', label='QPS')
.addYaxis(format='percentunit', min=0, label='Resident Hit Rate')
.addTarget(
  prometheus.target(
    'sum by (result) (rate(tici_reader_doc_cache_access_count{' + readerMetricMatcher + '}[1m]))',
    legendFormat='{{result}}',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_doc_cache_access_count{' + readerMetricMatcher + ', result="resident_hit"}[1m])) / sum(rate(tici_reader_doc_cache_access_count{' + readerMetricMatcher + '}[1m]))',
    legendFormat='resident-hit-rate',
  )
)
.addSeriesOverride({
  alias: 'resident-hit-rate',
  yaxis: 2,
});

local docCacheLoadLatencyPanel = graphPanel.new(
  title='Doc Cache Load Latency',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Latency of doc cache loads by load result.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_reader_doc_cache_load_latency_bucket{' + readerMetricMatcher + '}[1m])) by (le, result))',
    legendFormat='{{result}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_reader_doc_cache_load_latency_sum{' + readerMetricMatcher + '}[1m])) by (result) / sum(rate(tici_reader_doc_cache_load_latency_count{' + readerMetricMatcher + '}[1m])) by (result)',
    legendFormat='{{result}}-avg',
  )
);

// GRPC Row
local grpcRow = row.new(collapse=true, title='GRPC');
local grpcRequestDuration = graphPanel.new(
  title='Grpc Request Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_grpc_handle_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"}[1m])) by (le, type))',
    legendFormat='{{type}}-99%',
  )
);

local grpcRequestOPS = graphPanel.new(
  title='Grpc Request OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_grpc_handle_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local grpcRequestInflight = graphPanel.new(
  title='Grpc Request Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'sum by (service, type) (tici_grpc_handle_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"})',
    legendFormat='{{service}}-{{type}}',
  )
);

// Meta client row and its panels
local metaClientRow = row.new(collapse=true, title='Meta Client');

local metaClientGrpcDuration = graphPanel.new(
  title='Meta Client Grpc Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_meta_client_grpc_handle_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, type))',
    legendFormat='{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_meta_client_grpc_handle_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type) / sum(rate(tici_meta_client_grpc_handle_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (type)',
    legendFormat='{{type}}-avg',
  )
);

local metaClientGrpcOPS = graphPanel.new(
  title='Meta Client Grpc OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (type, result) (rate(tici_meta_client_grpc_handle_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{type}}-{{result}}',
  )
);

local metaClientGrpcInflight = graphPanel.new(
  title='Meta Client Grpc Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'sum by (type) (tici_meta_client_grpc_handle_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"})',
    legendFormat='{{type}}',
  )
);

local metaClientPoolFailedAttempts = graphPanel.new(
  title='Meta Client Pool Failed Attempts',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Failed attempts to get a meta client pool channel.',
)
.addTarget(
  prometheus.target(
    'sum by (instance) (rate(tici_meta_client_pool_failed_attempts{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{instance}}',
  )
);

local batchQueueRow = row.new(collapse=true, title='Batch Queue');

local batchQueueSubmitOPSPanel = graphPanel.new(
  title='Submit OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Metadata store batch queue submit rate by result.',
)
.addTarget(
  prometheus.target(
    'sum by (component, result) (rate(tici_batch_queue_submit_count{' + readerBatchQueueMatcher + '}[1m]))',
    legendFormat='{{component}}-{{result}}',
  )
);

local batchQueueRequestOPSPanel = graphPanel.new(
  title='Request OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Metadata store batch queue completion rate and merged-request rate.',
)
.addTarget(
  prometheus.target(
    'sum by (component, result) (rate(tici_batch_queue_request_count{' + readerBatchQueueMatcher + '}[1m]))',
    legendFormat='done-{{component}}-{{result}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (rate(tici_batch_queue_merged_request_count{' + readerBatchQueueMatcher + '}[1m]))',
    legendFormat='merged-{{component}}',
  )
);

local batchQueueQueueDurationPanel = graphPanel.new(
  title='Queue Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Wait time before enqueue and before execution.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_enqueue_block_duration_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='enqueue-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_enqueue_block_duration_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_enqueue_block_duration_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='enqueue-{{component}}-avg',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_queue_duration_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='queue-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_queue_duration_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_queue_duration_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='queue-{{component}}-avg',
  )
);

local batchQueueRequestDurationPanel = graphPanel.new(
  title='Request Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='End-to-end request latency from submit to response.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_request_duration_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_request_duration_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_request_duration_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='{{component}}-avg',
  )
);

local batchQueueBatchDurationPanel = graphPanel.new(
  title='Batch Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Batch collection and execution duration.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_collect_duration_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='collect-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_collect_duration_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_collect_duration_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='collect-{{component}}-avg',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_execute_duration_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='execute-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_execute_duration_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_execute_duration_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='execute-{{component}}-avg',
  )
);

local batchQueueBacklogPanel = graphPanel.new(
  title='Backlog',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Queued request and weight backlog.',
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_request_backlog{' + readerBatchQueueMatcher + '})',
    legendFormat='request-{{component}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_weight_backlog{' + readerBatchQueueMatcher + '})',
    legendFormat='weight-{{component}}',
  )
);

local batchQueueInflightPanel = graphPanel.new(
  title='Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Inflight submit calls, running batches, and running requests.',
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_submit_inflight{' + readerBatchQueueMatcher + '})',
    legendFormat='submit-{{component}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_batch_inflight{' + readerBatchQueueMatcher + '})',
    legendFormat='batch-{{component}}',
  )
)
.addTarget(
  prometheus.target(
    'sum by (component) (tici_batch_queue_request_inflight{' + readerBatchQueueMatcher + '})',
    legendFormat='request-{{component}}',
  )
);

local batchQueueBatchShapePanel = graphPanel.new(
  title='Batch Shape',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Batch size in requests and accumulated batch weight.',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_size_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='size-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_size_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_size_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='size-{{component}}-avg',
  )
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_batch_queue_batch_weight_bucket{' + readerBatchQueueMatcher + '}[1m])) by (le, component))',
    legendFormat='weight-{{component}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_batch_queue_batch_weight_sum{' + readerBatchQueueMatcher + '}[1m])) by (component) / sum(rate(tici_batch_queue_batch_weight_count{' + readerBatchQueueMatcher + '}[1m])) by (component)',
    legendFormat='weight-{{component}}-avg',
  )
);

local batchQueueFlushOPSPanel = graphPanel.new(
  title='Flush OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Batch flush trigger rate by reason.',
)
.addTarget(
  prometheus.target(
    'sum by (component, reason) (rate(tici_batch_queue_batch_flush_count{' + readerBatchQueueMatcher + '}[1m]))',
    legendFormat='{{component}}-{{reason}}',
  )
);

// Dedup row and its panels
local dedupRow = row.new(collapse=true, title='Dedup');

local dedupRunOPS = graphPanel.new(
  title='Dedup Run OPS',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
)
.addTarget(
  prometheus.target(
    'sum by (result) (rate(tici_dedup_run_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m]))',
    legendFormat='{{result}}',
  )
);

local dedupInProgress = graphPanel.new(
  title='Dedup In Progress',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
)
.addTarget(
  prometheus.target(
    'tici_dedup_in_progress{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}',
    legendFormat='{{instance}}',
  )
);

local dedupStageLatency = graphPanel.new(
  title='Dedup Stage Latency',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_dedup_stage_latency_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (le, stage))',
    legendFormat='{{stage}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_dedup_stage_latency_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage) / sum(rate(tici_dedup_stage_latency_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance"}[1m])) by (stage)',
    legendFormat='{{stage}}-avg',
  )
);

// Storage row and its panels
local storageRow = row.new(collapse=true, title='Storage');

local storageHandleDuration = graphPanel.new(
  title='Storage Handle Duration',
  datasource=myDS,
  legend_rightSide=true,
  format='s',
  description='Backend storage request duration by backend and operation type',
)
.addTarget(
  prometheus.target(
    'histogram_quantile(0.99, sum(rate(tici_storage_handle_duration_bucket{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"}[1m])) by (le, backend, type))',
    legendFormat='{{backend}}/{{type}}-99',
  )
)
.addTarget(
  prometheus.target(
    'sum(rate(tici_storage_handle_duration_sum{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"}[1m])) by (backend, type) / sum(rate(tici_storage_handle_duration_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"}[1m])) by (backend, type)',
    legendFormat='{{backend}}/{{type}}-avg',
  )
);

local storageHandleCount = graphPanel.new(
  title='Storage Handle Count',
  datasource=myDS,
  legend_rightSide=true,
  format='ops',
  description='Backend storage request rate by backend, operation type, and result',
)
.addTarget(
  prometheus.target(
    'sum by (backend, type, result) (rate(tici_storage_handle_count{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"}[1m]))',
    legendFormat='{{backend}}/{{type}}-{{result}}',
  )
);

local storageHandleInflight = graphPanel.new(
  title='Storage Handle Inflight',
  datasource=myDS,
  legend_rightSide=true,
  format='none',
  description='Inflight backend storage requests by backend and operation type',
)
.addTarget(
  prometheus.target(
    'sum by (backend, type) (tici_storage_handle_inflight{k8s_cluster="$k8s_cluster", tidb_cluster="$tidb_cluster", instance=~"$instance", service="reader"})',
    legendFormat='{{backend}}/{{type}}',
  )
);

// Merge together.
local panelW = 12;
local panelH = 7;
local rowW = 24;
local rowH = 1;

local rowPos = {x:0, y:0, w:rowW, h:rowH};
local fullPanelPos = {x:0, y:0, w:rowW, h:panelH};
local leftPanelPos = {x:0, y:0, w:panelW, h:panelH};
local rightPanelPos = {x:panelW, y:0, w:panelW, h:panelH};

newDash
.addPanel(
  serverRow
  .addPanel(uptimeP, gridPos=leftPanelPos)
  .addPanel(cpuP, gridPos=rightPanelPos)
  .addPanel(memP, gridPos=leftPanelPos)
  .addPanel(processMemoryStatusPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  shardRow
  .addPanel(CountPanel, gridPos=leftPanelPos)
  .addPanel(metadataScanSkippedPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  warmupRow
  .addPanel(warmupDurationPanel, gridPos=fullPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  heartbeatRow
  .addPanel(shardRefreshDurationPanel, gridPos=leftPanelPos)
  .addPanel(shardRefreshCountPanel, gridPos=rightPanelPos)
  .addPanel(shardRefreshQueueDurationPanel, gridPos=leftPanelPos)
  .addPanel(heartbeatQueueBacklogPanel, gridPos=rightPanelPos)
  .addPanel(shardRefreshInflightPanel, gridPos=leftPanelPos)
  .addPanel(shardRefreshEventCountPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  queryRow
  .addPanel(queryLatencyPanel, gridPos=leftPanelPos)
  .addPanel(shardQueryDurationPanel, gridPos=leftPanelPos)
  .addPanel(shardQueryCountPanel, gridPos=rightPanelPos)
  .addPanel(shardQueryOpenDurationPanel, gridPos=leftPanelPos)
  .addPanel(shardQueryNextDurationPanel, gridPos=rightPanelPos)
  .addPanel(shardQueryBatchCountPanel, gridPos=leftPanelPos)
  .addPanel(shardQueryInflightPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  docCacheRow
  .addPanel(docCacheAccessPanel, gridPos=leftPanelPos)
  .addPanel(docCacheLoadLatencyPanel, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  grpcRow
  .addPanel(grpcRequestDuration, gridPos=leftPanelPos)
  .addPanel(grpcRequestOPS, gridPos=rightPanelPos)
  .addPanel(grpcRequestInflight, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  metaClientRow
  .addPanel(metaClientGrpcDuration, gridPos=leftPanelPos)
  .addPanel(metaClientGrpcOPS, gridPos=rightPanelPos)
  .addPanel(metaClientGrpcInflight, gridPos=leftPanelPos)
  .addPanel(metaClientPoolFailedAttempts, gridPos=rightPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  batchQueueRow
  .addPanel(batchQueueSubmitOPSPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueRequestOPSPanel, gridPos=rightPanelPos)
  .addPanel(batchQueueQueueDurationPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueRequestDurationPanel, gridPos=rightPanelPos)
  .addPanel(batchQueueBatchDurationPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueBacklogPanel, gridPos=rightPanelPos)
  .addPanel(batchQueueInflightPanel, gridPos=leftPanelPos)
  .addPanel(batchQueueBatchShapePanel, gridPos=rightPanelPos)
  .addPanel(batchQueueFlushOPSPanel, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  dedupRow
  .addPanel(dedupRunOPS, gridPos=leftPanelPos)
  .addPanel(dedupInProgress, gridPos=rightPanelPos)
  .addPanel(dedupStageLatency, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)
.addPanel(
  storageRow
  .addPanel(storageHandleDuration, gridPos=leftPanelPos)
  .addPanel(storageHandleCount, gridPos=rightPanelPos)
  .addPanel(storageHandleInflight, gridPos=leftPanelPos)
  ,
  gridPos=rowPos
)