{
  "dashboard_title": "Dedicated Dataplane Node Info",
  "rows": [
    {
      "title": null,
      "panels": [
        {
          "title": "Notice",
          "subtitle": "Choose \"namespace\" to show DM metrics.\nChoose \"pod\" to switch node instance.",
          "expressions": []
        }
      ]
    },
    {
      "title": "Node Info",
      "panels": [
        {
          "title": "Node Info",
          "subtitle": null,
          "expressions": [
            "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\",  \"$0\", \"instance\", \".*\") * on(node,k8s_cluster_info) group_right kube_node_info"
          ]
        },
        {
          "title": "Node labels",
          "subtitle": null,
          "expressions": [
            "label_replace(node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}, \"node\",  \"$0\", \"instance\", \".*\") * on(node, k8s_cluster_info) group_right sum(kube_node_labels) by (node, k8s_cluster_info, label_zone, label_dedicated, label_failure_domain_beta_kubernetes_io_zone, label_beta_kubernetes_io_instance_type)",
            "(\n  label_replace(\n    node_uname_info{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"},\n    \"node\", \"$0\", \"instance\", \".*\"\n  )\n  * on(node, k8s_cluster_info) group_right\n    sum(kube_node_labels)\n      by (node, k8s_cluster_info,\n          label_zone, label_dedicated,\n          label_failure_domain_beta_kubernetes_io_zone,\n          label_beta_kubernetes_io_instance_type)\n)\n* on(node) group_left(pod, component)\n  label_replace(\n    kube_pod_info{namespace=~\"$namespace\", pod=~\"$pod\"},\n    \"component\", \"$2\", \"pod\",\n    \"(^|.*-)(tidb|tikv|tiflash|pd|tso|ticdc)(-[0-9a-z]+)*$\"\n  )"
          ]
        },
        {
          "title": "Component Node Number Trends",
          "subtitle": null,
          "expressions": [
            "count(\n  label_replace(\n    kube_pod_info{namespace=\"$namespace\", pod=~\"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"},\n    \"component\", \n    \"$1$3\", \n    \"pod\", \n    \"db-.*(tidb|tikv|tiflash)(.*)|db-[0-9]+-[a-z0-9]+-(tidb|tikv|tiflash)-[0-9]+\"\n  )\n) by (component)"
          ]
        }
      ]
    },
    {
      "title": "Network",
      "panels": [
        {
          "title": "Network Received",
          "subtitle": null,
          "expressions": [
            "rate(node_network_receive_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])"
          ]
        },
        {
          "title": "Network Received Drop",
          "subtitle": null,
          "expressions": [
            "rate(node_network_receive_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])"
          ]
        },
        {
          "title": "Network Transmitted",
          "subtitle": null,
          "expressions": [
            "rate(node_network_transmit_bytes_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])"
          ]
        },
        {
          "title": "Network Transmitted Drop",
          "subtitle": null,
          "expressions": [
            "rate(node_network_transmit_drop_total{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[1m])"
          ]
        },
        {
          "title": "bw_in_allowance_exceeded",
          "subtitle": "the number of packets that are queued or dropped because of instance bandwidth exceeded allowance for the instance.",
          "expressions": [
            "increase(node_ethtool_bw_in_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])"
          ]
        },
        {
          "title": "bw_out_allowance_exceeded",
          "subtitle": "the number of packets that are queued or dropped because of instance bandwidth exceeded allowance for the instance.",
          "expressions": [
            "increase(node_ethtool_bw_out_allowance_exceeded{cluster=\"$k8s_cluster\", instance=~\"$instance\"}[1m])"
          ]
        },
        {
          "title": "linklocal_allowance_exceeded",
          "subtitle": "linklocal_allowance_exceeded indicates the number of packets dropped due to PPS rate allowance exceeded for local services such as Route 53 DNS Resolver, Instance Metadata Service, Amazon Time Sync Service. This often points to suboptimal design choices or misconfiguration. This allowance is the same across instances.",
          "expressions": [
            "increase(node_ethtool_linklocal_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])"
          ]
        },
        {
          "title": "PPS_allowance_exceeded",
          "subtitle": "number of packets queued or dropped due to Packet Per Second (PPS) exceeding the allowance for the instance. ",
          "expressions": [
            "increase(node_ethtool_pps_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])"
          ]
        },
        {
          "title": "conntrack_allowance_exceeded",
          "subtitle": "the number of packets dropped due to exhaustion of tracked session allowance for the instance, new sessions will fail to establish once this allowance is exceeded.",
          "expressions": [
            "increase(node_ethtool_conntrack_allowance_exceeded{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[5m])"
          ]
        },
        {
          "title": "Conntrack Table Utilization",
          "subtitle": null,
          "expressions": [
            "node_nf_conntrack_entries{cluster=\"$k8s_cluster\",instance=~\"$instance\"} / node_nf_conntrack_entries_limit{cluster=\"$k8s_cluster\",instance=~\"$instance\"}"
          ]
        },
        {
          "title": "Conntrack Drops",
          "subtitle": null,
          "expressions": [
            "rate(node_nf_conntrack_stat_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])"
          ]
        },
        {
          "title": "Conntrack Insert Failures",
          "subtitle": null,
          "expressions": [
            "rate(node_nf_conntrack_stat_insert_failed{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])"
          ]
        },
        {
          "title": "Conntrack Early Drops",
          "subtitle": null,
          "expressions": [
            "rate(node_nf_conntrack_stat_early_drop{cluster=\"$k8s_cluster\",instance=~\"$instance\"}[1m])"
          ]
        }
      ]
    },
    {
      "title": "Memory",
      "panels": [
        {
          "title": "Memory Usage",
          "subtitle": null,
          "expressions": [
            "max(\n  node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n  - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n  - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n  - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n)\n",
            "max(node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})",
            "max(node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})",
            "max(node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$node:$port\", k8s_cluster_info=\"$k8s_cluster\"})"
          ]
        },
        {
          "title": "Memory Usage",
          "subtitle": null,
          "expressions": [
            "max(\n  (\n    (\n      node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n    - node_memory_MemFree_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n    - node_memory_Buffers_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n    - node_memory_Cached_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n    )\n    / node_memory_MemTotal_bytes{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}\n  ) * 100)\n"
          ]
        }
      ]
    },
    {
      "title": "CPU",
      "panels": [
        {
          "title": "System load",
          "subtitle": null,
          "expressions": [
            "max(node_load1{job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})",
            "max(node_load5{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})",
            "max(node_load15{ job=\"node-exporter\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})"
          ]
        },
        {
          "title": "Usage Per Core",
          "subtitle": null,
          "expressions": [
            "sum by (cpu) (irate(node_cpu_seconds_total{ job=\"node-exporter\", mode!=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m]))"
          ]
        },
        {
          "title": "CPU Utilization",
          "subtitle": null,
          "expressions": [
            "100 - (avg by (instance) (irate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[5m])) * 100)"
          ]
        }
      ]
    },
    {
      "title": "Disk",
      "panels": [
        {
          "title": "Disk I/O Util",
          "subtitle": null,
          "expressions": [
            "rate(node_disk_io_time_seconds_total{ job=\"node-exporter\",  instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Read Disk iops",
          "subtitle": null,
          "expressions": [
            "rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])",
            "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}[2m])"
          ]
        },
        {
          "title": "Write Disk iops",
          "subtitle": null,
          "expressions": [
            "rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Read Disk Wait Latency",
          "subtitle": null,
          "expressions": [
            "rate(node_disk_read_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])",
            "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Write Disk Wait Latency",
          "subtitle": null,
          "expressions": [
            "rate(node_disk_write_time_seconds_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m]) / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Read Disk Bandwidth",
          "subtitle": "Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited.  Amount of data being written to the disk can be used to estimate Flash storage life time.",
          "expressions": [
            "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])",
            "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Write Disk Bandwidth",
          "subtitle": "Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited.  Amount of data being written to the disk can be used to estimate Flash storage life time.",
          "expressions": [
            "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Read Disk IO Avg Size / second",
          "subtitle": "Shows average size of a single disk operation.",
          "expressions": [
            "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])",
            "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Write Disk IO Avg Size / second",
          "subtitle": "Shows average size of a single disk operation.",
          "expressions": [
            "rate(node_disk_read_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_reads_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])",
            "rate(node_disk_written_bytes_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m]) * 512 / rate(node_disk_writes_completed_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device!~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Read Disk (wrqm & rrqm) / second",
          "subtitle": "Shows how effectively Operating System is able to merge logical IO requests into physical requests.  This is a good measure of the IO locality which can be used for workload characterization.",
          "expressions": [
            "rate(node_disk_reads_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~ \"dm.*\"}[2m])",
            "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Disk (wrqm & rrqm) / second",
          "subtitle": "Shows how effectively Operating System is able to merge logical IO requests into physical requests.  This is a good measure of the IO locality which can be used for workload characterization.",
          "expressions": [
            "rate(node_disk_writes_merged_total{instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\", device !~\"dm.*\"}[2m])"
          ]
        },
        {
          "title": "Disk Space Usage",
          "subtitle": null,
          "expressions": [
            "((max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) /\n max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"})) * 100"
          ]
        },
        {
          "title": "Disk Space Used",
          "subtitle": null,
          "expressions": [
            "(max by(device, instance) (node_filesystem_size_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}) -\n max by(device, instance) (node_filesystem_avail_bytes{device!~\"rootfs|tmpfs\", instance=~\"$instance\", k8s_cluster_info=\"$k8s_cluster\"}))"
          ]
        },
        {
          "title": "TiKV/TiFlash PVC inode usage",
          "subtitle": "Show the inode usage of tikv and tiflash PVC ",
          "expressions": [
            " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}"
          ]
        },
        {
          "title": "TiDB PVC inode usage",
          "subtitle": "Show the inode usage of tikv and tiflash PVC ",
          "expressions": [
            " kubelet_volume_stats_inodes_used{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}/kubelet_volume_stats_inodes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}"
          ]
        },
        {
          "title": "TiKV/TiFlash PVC disk usage",
          "subtitle": "Show the inode usage of tikv and tiflash PVC ",
          "expressions": [
            "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tikv.*|.*tiflash.*\", k8s_cluster_info=\"$k8s_cluster\"}"
          ]
        },
        {
          "title": "TiDB PVC disk usage",
          "subtitle": "Show the inode usage of tikv and tiflash PVC ",
          "expressions": [
            "kubelet_volume_stats_capacity_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"} - kubelet_volume_stats_available_bytes{namespace=\"$namespace\", persistentvolumeclaim=~\".*tidb.*\", k8s_cluster_info=\"$k8s_cluster\"}"
          ]
        }
      ]
    },
    {
      "title": "EBS Volume Performance",
      "panels": [
        {
          "title": "EBS Volume IO Exceeded ",
          "expressions": [
            "rate(aws_ebs_csi_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])",
            "rate(aws_ebs_csi_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])",
            "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{instance=~\"$instance\"}[$__rate_interval])",
            "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{instance=~\"$instance\"}[$__rate_interval])"
          ]
        },
        {
          "title": "Read IOPS",
          "subtitle": "It represents the read IOPS performed on EBS volumes. For more info, please refer to https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html#nvme-stats",
          "expressions": [
            "rate(aws_ebs_csi_read_ops_total{instance=~\"$instance\"}[$__rate_interval])"
          ]
        },
        {
          "title": "Write IOPS",
          "subtitle": "It represents the write IOPS performed on EBS volumes. For more info, please refer to https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html#nvme-stats",
          "expressions": [
            "rate(aws_ebs_csi_write_ops_total{instance=~\"$instance\"}[$__rate_interval])"
          ]
        },
        {
          "title": "Read IO Latency",
          "subtitle": "This calculates the 99th percentile of EBS read I/O latency. It measures worst-case read latency. For more info, please refer to https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html#nvme-stats",
          "expressions": [
            "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))",
            "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_read_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))"
          ]
        },
        {
          "title": "Write IO Latency",
          "subtitle": "This calculates the 99th percentile of EBS write I/O latency. It measures worst-case write latency. For more info, please refer to https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html#nvme-stats",
          "expressions": [
            "histogram_quantile(0.99, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))",
            "histogram_quantile(1, sum by(le, instance, volume_id) (rate(aws_ebs_csi_write_io_latency_seconds_bucket{instance=~\"$instance\"}[$__rate_interval])))"
          ]
        },
        {
          "title": "Read Throughput",
          "subtitle": "Read Throughput of the EBS volumes.\nFor more info, please refer to https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html#nvme-stats",
          "expressions": [
            "rate(aws_ebs_csi_read_bytes_total{instance=~\"$instance\"}[$__rate_interval])"
          ]
        },
        {
          "title": "Write Throughput",
          "subtitle": "Write Throughput of the EBS volumes\nFor more info, please refer to https://docs.aws.amazon.com/ebs/latest/userguide/nvme-detailed-performance-stats.html#nvme-stats",
          "expressions": [
            "rate(aws_ebs_csi_write_bytes_total{instance=~\"$instance\"}[$__rate_interval])"
          ]
        }
      ]
    },
    {
      "title": "EBS Volume Performance - Cluster latitude",
      "panels": [
        {
          "title": "TiDB Cluster Pods",
          "subtitle": null,
          "expressions": [
            "kube_pod_info{namespace=\"$namespace\"}"
          ]
        },
        {
          "title": "EBS Volume IO Exceeded ",
          "expressions": [
            "rate(aws_ebs_csi_exceeded_iops_seconds_total{}[$__rate_interval])",
            "rate(aws_ebs_csi_exceeded_tp_seconds_total{}[$__rate_interval])",
            "rate(aws_ebs_csi_ec2_exceeded_tp_seconds_total{}[$__rate_interval])",
            "rate(aws_ebs_csi_ec2_exceeded_iops_seconds_total{}[$__rate_interval])"
          ]
        },
        {
          "title": "Disk wait latency",
          "subtitle": null,
          "expressions": [
            "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n  kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n  \"instance\", \"$1\", \"node\", \"(.*)\"\n)",
            "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n  kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv|tiflash).*\"},\n  \"instance\", \"$1\", \"node\", \"(.*)\"\n)"
          ]
        },
        {
          "title": "Disk read wait latency (TiKV)",
          "subtitle": null,
          "expressions": [
            "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n  kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n  \"instance\", \"$1\", \"node\", \"(.*)\"\n)"
          ]
        },
        {
          "title": "Disk write wait latency (TiKV)",
          "subtitle": null,
          "expressions": [
            "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n  kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tikv).*\"},\n  \"instance\", \"$1\", \"node\", \"(.*)\"\n)"
          ]
        },
        {
          "title": "Disk read wait latency (TiFlash)",
          "subtitle": null,
          "expressions": [
            "(\nrate(node_disk_read_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_reads_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(pod)\nlabel_replace(\n  kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n  \"instance\", \"$1\", \"node\", \"(.*)\"\n)"
          ]
        },
        {
          "title": "Disk write wait latency (TiFlash)",
          "subtitle": null,
          "expressions": [
            "(\nrate(node_disk_write_time_seconds_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m]) \n/ \nrate(node_disk_writes_completed_total{k8s_cluster_info=\"$k8s_cluster\", device!~\"dm-.*|nvme1.*|nvme0.*\"}[2m])\n)\n* on(instance)\ngroup_left(node)\nlabel_replace(\n  kube_pod_info{namespace=~\"$namespace\", pod=~\".*(tiflash).*\"},\n  \"instance\", \"$1\", \"node\", \"(.*)\"\n)"
          ]
        },
        {
          "title": "Disk Bandwidth",
          "subtitle": "Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited.  Amount of data being written to the disk can be used to estimate Flash storage life time.",
          "expressions": [
            "rate(node_disk_read_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])",
            "rate(node_disk_written_bytes_total{k8s_cluster_info=\"$k8s_cluster\"}[2m])"
          ]
        }
      ]
    },
    {
      "title": "TCP",
      "panels": [
        {
          "title": "TCP In Use",
          "subtitle": null,
          "expressions": [
            "node_sockstat_TCP_inuse{instance=~\"$instance\"}"
          ]
        },
        {
          "title": "Segments retransmitted",
          "subtitle": null,
          "expressions": [
            "rate(node_netstat_Tcp_RetransSegs{instance=~\"$instance\"}[1m])"
          ]
        },
        {
          "title": "TCP Connections",
          "subtitle": null,
          "expressions": [
            "node_netstat_Tcp_CurrEstab{instance=~\"$instance\"}"
          ]
        }
      ]
    }
  ]
}