groups:
- name: node-exporter.rules
  rules:
  - expr: |
      count without (cpu) (
        count without (mode) (
          node_cpu_seconds_total{job="node-exporter"}
        )
      )
    record: instance:node_num_cpu:sum
  - expr: |
      1 - avg without (cpu, mode) (
        rate(node_cpu_seconds_total{job="node-exporter", mode="idle"}[1m])
      )
    record: instance:node_cpu_utilisation:rate1m
  - expr: |
      (
        node_load1{job="node-exporter"}
      /
        instance:node_num_cpu:sum{job="node-exporter"}
      )
    record: instance:node_load1_per_cpu:ratio
  - expr: |
      1 - (
        node_memory_MemAvailable_bytes{job="node-exporter"}
      /
        node_memory_MemTotal_bytes{job="node-exporter"}
      )
    record: instance:node_memory_utilisation:ratio
  - expr: |
      rate(node_vmstat_pgmajfault{job="node-exporter"}[1m])
    record: instance:node_vmstat_pgmajfault:rate1m
  - expr: |
      rate(node_disk_io_time_seconds_total{job="node-exporter", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m])
    record: instance_device:node_disk_io_time_seconds:rate1m
  - expr: |
      rate(node_disk_io_time_weighted_seconds_total{job="node-exporter", device=~"nvme.+|rbd.+|sd.+|vd.+|xvd.+|dm-.+|dasd.+"}[1m])
    record: instance_device:node_disk_io_time_weighted_seconds:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_receive_bytes_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_receive_bytes_excluding_lo:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_transmit_bytes_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_transmit_bytes_excluding_lo:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_receive_drop_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_receive_drop_excluding_lo:rate1m
  - expr: |
      sum without (device) (
        rate(node_network_transmit_drop_total{job="node-exporter", device!="lo"}[1m])
      )
    record: instance:node_network_transmit_drop_excluding_lo:rate1m

  - alert: NodeFilesystemSpaceFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available space left and is filling
        up.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
      summary: Filesystem free space is less than 30%.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!~"erofs|"} / node_filesystem_size_bytes{job="node-exporter",fstype!~"erofs|"} * 100 < 30
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: AdminNodeFilesystemSpaceFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available space left and is filling
        up fast.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
      summary: Filesystem free space is less than {{ printf "%.2f" $value }}%.
    expr: |
      # dedicated AWS admin node, threshold 15%
      (
        label_replace(
          (
            node_filesystem_avail_bytes{job="node-exporter", fstype!~"erofs|"} /
            node_filesystem_size_bytes{job="node-exporter", fstype!~"erofs|"} and
            node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
          ),
          "node","$1","instance", "(.+)"
        )
        * on (cluster, node) group_left()
        kube_node_labels{seed_provider="aws", label_tidbcloud_com_component="shared", provider_type!~"aws-free-tier"}
      ) * 100 < 15
      or
      # dedicated GCP admin node, threshold 15%
      (
        label_replace(
          (
            node_filesystem_avail_bytes{job="node-exporter", fstype!~"erofs|"} /
            node_filesystem_size_bytes{job="node-exporter", fstype!~"erofs|"} and
            node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
          ),
          "node","$1","instance", "(.+)"
        )
        * on (cluster, node) group_left()
        kube_node_labels{seed_provider="gcp", label_usedby="dbaas-control-plane-shoot", provider_type!~"aws-free-tier"}
      ) * 100 < 15
    for: 10m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: TiDBComponentNodeFilesystemSpaceFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available space left and is filling
        up fast.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
      summary: Filesystem free space is less than {{ printf "%.2f" $value }}%.
    expr: |
      # dedicated AWS worker node, threshold 20%
      (
        label_replace(
          (
            node_filesystem_avail_bytes{job="node-exporter", fstype!~"erofs|"} /
            node_filesystem_size_bytes{job="node-exporter", fstype!~"erofs|"} and
            node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
          ),
          "node","$1","instance", "(.+)"
        )
        * on (cluster, node) group_left()
        kube_node_labels{seed_provider="aws", label_tidbcloud_com_component!="shared", provider_type!~"aws-free-tier"}
      ) * 100 < 20
      or
      # dedicated GCP worker node, threshold 25%
      (
        label_replace(
          (
            node_filesystem_avail_bytes{job="node-exporter", fstype!~"erofs|"} /
            node_filesystem_size_bytes{job="node-exporter", fstype!~"erofs|"} and
            node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
          ),
          "node","$1","instance", "(.+)"
        )
        * on (cluster, node) group_left()
        kube_node_labels{seed_provider="gcp", label_usedby!="dbaas-control-plane-shoot", provider_type!~"aws-free-tier"}
      ) * 100 < 25
    for: 3m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: ServerlessNodeFilesystemSpaceFillingUp
    annotations:
      message: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ $value }}% available space left and is filling
        up fast.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
      summary: Filesystem free space is less than {{ printf "%.2f" $value }}%.
    expr: |
      # serverless node, threshold 15%
      (
        label_replace(
          (
            node_filesystem_avail_bytes{job="node-exporter", fstype!~"erofs|"} /
            node_filesystem_size_bytes{job="node-exporter", fstype!~"erofs|"} and
            node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
          ),
          "node","$1","instance", "(.+)"
        )
        * on (cluster, node) group_left()
        kube_node_labels{provider_type=~"aws-free-tier|alicloud-serverless"}
      ) * 100 < 15
    for: 10m
    labels:
      component: node
      oncall_team: serverless
      severity: major
      response_class: ticket
  - alert: NodeFilesystemAlmostOutOfSpace
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available space left.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
      summary: Filesystem has less than 5% space left.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!~"erofs|"} / node_filesystem_size_bytes{job="node-exporter",fstype!~"erofs|"} * 100 < 5
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeFilesystemAlmostOutOfSpace
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available space left.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
      summary: Filesystem has less than 3% space left.
    expr: |
      (
        node_filesystem_avail_bytes{job="node-exporter",fstype!~"erofs|"} / node_filesystem_size_bytes{job="node-exporter",fstype!~"erofs|"} * 100 < 3
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: NodeFilesystemFilesFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available inodes left and is filling
        up.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup
      summary: Filesystem is predicted to run out of inodes within the next 24 hours.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!~"erofs|"} / node_filesystem_files{job="node-exporter",fstype!~"erofs|"} * 100 < 40
      and
        predict_linear(node_filesystem_files_free{job="node-exporter",fstype!~"erofs|"}[6h], 24*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeFilesystemFilesFillingUp
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available inodes left and is filling
        up fast.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup
      summary: Filesystem is predicted to run out of inodes within the next 4 hours.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!~"erofs|"} / node_filesystem_files{job="node-exporter",fstype!~"erofs|"} * 100 < 20
      and
        predict_linear(node_filesystem_files_free{job="node-exporter",fstype!~"erofs|"}[6h], 4*60*60) < 0
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: NodeFilesystemAlmostOutOfFiles
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available inodes left.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles
      summary: Filesystem has less than 5% inodes left.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!~"erofs|"} / node_filesystem_files{job="node-exporter",fstype!~"erofs|"} * 100 < 5
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeFilesystemAlmostOutOfFiles
    annotations:
      description: Filesystem on {{ $labels.device }} at {{ $labels.instance }}
        has only {{ printf "%.2f" $value }}% available inodes left.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles
      summary: Filesystem has less than 3% inodes left.
    expr: |
      (
        node_filesystem_files_free{job="node-exporter",fstype!~"erofs|"} / node_filesystem_files{job="node-exporter",fstype!~"erofs|"} * 100 < 3
      and
        node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
      )
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
  - alert: NodeNetworkReceiveErrs
    annotations:
      description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
        {{ printf "%.0f" $value }} receive errors in the last two minutes.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworkreceiveerrs
      summary: Network interface is reporting many receive errors.
    expr: |
      increase(node_network_receive_errs_total[2m]) > 10
    for: 1h
    labels:
      severity: warning
      component: node
      oncall_team: dedicated_service
  - alert: NodeNetworkTransmitErrs
    annotations:
      description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered
        {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworktransmiterrs
      summary: Network interface is reporting many transmit errors.
    expr: |
      increase(node_network_transmit_errs_total[2m]) > 10
    for: 1h
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeHighNumberConntrackEntriesUsed
    annotations:
      description: '{{ $value | humanizePercentage }} of conntrack entries are used'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodehighnumberconntrackentriesused
      summary: Number of conntrack are getting close to the limit
    expr: |
      (node_nf_conntrack_entries / node_nf_conntrack_entries_limit) > 0.75
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeClockSkewDetected
    annotations:
      message: Clock on {{ $labels.instance }} is out of sync by more than 300s.
        Ensure NTP is configured correctly on this host.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclockskewdetected
      summary: Clock skew detected.
    expr: |
      (
        node_timex_offset_seconds > 0.05
      and
        deriv(node_timex_offset_seconds[5m]) >= 0
      )
      or
      (
        node_timex_offset_seconds < -0.05
      and
        deriv(node_timex_offset_seconds[5m]) <= 0
      )
    for: 10m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeClockNotSynchronising
    annotations:
      message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP
        is configured on this host.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclocknotsynchronising
      summary: Clock not synchronising.
    expr: |
      min_over_time(node_timex_sync_status[5m]) == 0
    for: 10m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
  - alert: NodeExporterServerIsDown
    expr: up{component="node-exporter"} == 0
    for: 10m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
      expr: up{component="node-exporter"} == 0
    annotations:
      description: 'cluster: {{ $labels.kubernetes_namespace }}, instance: {{ $labels.instance }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.lh79z51iefp5
      value: '{{ $value }}'
      summary: "Node Exporter server is down (instance {{ $labels.instance }})"
  - alert: NodeDiskLowSpace
    expr: node_filesystem_avail_bytes{fstype!~"erofs|", mountpoint!="/boot"} / node_filesystem_size_bytes{fstype!~"erofs|", mountpoint!="/boot"} * 100 <= 15
    for: 5m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
      expr: node_filesystem_avail_bytes{fstype!~"erofs|", mountpoint!="/boot"} / node_filesystem_size_bytes{fstype!~"erofs|",mountpoint!="/boot"} * 100 <= 15
    annotations:
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.pysrqzxka28r
      value: '{{ $value }}'
      summary: "Host disk space usage is more than 85% (instance {{ $labels.instance }})"
  - alert: NodeDiskVeryLowSpace
    expr: node_filesystem_avail_bytes{fstype!~"erofs|" ,mountpoint!="/boot",device!="/dev/root"} / node_filesystem_size_bytes{fstype!~"erofs|" ,mountpoint!="/boot",device!="/dev/root"} * 100 <= 10
    for: 2m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
      expr: node_filesystem_avail_bytes{fstype!~"erofs|"} / node_filesystem_size_bytes{fstype!~"erofs|"} * 100 <= 10
    annotations:
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.me9q3m5ke8hr
      value: '{{ $value }}'
      summary: "Host disk space usage is more than 90% (instance {{ $labels.instance }})"
  - alert: NodeDiskLowSpaceWeTechUsSaaS
    expr: |
        label_replace(node_filesystem_avail_bytes{fstype!~"erofs|", mountpoint!="/boot", tenant="1372813089209061633",device!="/dev/root"}, "node", "$1", "instance", "(.*)")
        / label_replace(node_filesystem_size_bytes{fstype!~"erofs|", mountpoint!="/boot", tenant="1372813089209061633",device!="/dev/root"}, "node", "$1", "instance", "(.*)")
        * on(node,k8s_cluster_info) group_left(label_component) kube_node_labels{tenant="1372813089209061633",label_component!="backup",label_tidbcloud_com_component!="shared"} * 100 <= 20
    for: 2m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
    annotations:
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.me9q3m5ke8hr
      value: '{{ $value }}'
      summary: "Host disk space usage on tidbcluster node is more than 80% (instance {{ $labels.instance }})"
  - alert: AdminNodeDiskLowSpaceWeTechUsSaaS
    expr: |
        label_replace(node_filesystem_avail_bytes{fstype!~"erofs|", mountpoint!="/boot", tenant="1372813089209061633",device!="/dev/root"}, "node", "$1", "instance", "(.*)")
        / label_replace(node_filesystem_size_bytes{fstype!~"erofs|", mountpoint!="/boot", tenant="1372813089209061633",device!="/dev/root"}, "node", "$1", "instance", "(.*)")
        * on(node,k8s_cluster_info) group_left(label_component) kube_node_labels{tenant="1372813089209061633",label_component!="backup",label_tidbcloud_com_component="shared"} * 100 <= 20
    for: 2m
    labels:
      component: dedicated-regional-server
      oncall_team: dedicated_service
      severity: major
      response_class: ticket
    annotations:
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }}, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.me9q3m5ke8hr
      value: '{{ $value }}'
      summary: "Host disk space usage on admin node is more than 80% (instance {{ $labels.instance }})"
      solution: "This may be caused by log files or other temporary files taking up too much space. Please try 'dd if=/dev/zero of=/var/lib/my_zero_file.img bs=1M count=1024' to increase disk usage to trigger k8s GC."
  - alert: NodeDiskLowAvailableInode
    expr: node_filesystem_files_free{fstype!~"erofs|"} / node_filesystem_files{fstype!~"erofs|"} * 100 < 20
    for: 5m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
      expr: node_filesystem_files_free{fstype!~"erofs|"} / node_filesystem_files{fstype!~"erofs|"} * 100 < 20
    annotations:
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }} disk inode usage more than 80%, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.1laamqn6yrdq
      value: '{{ $value }}'
      summary: "Host disk inode usage is more than 80% (instance {{ $labels.instance }})"
  - alert: NodeDiskVeryLowAvailableInode
    expr: node_filesystem_files_free{fstype!~"erofs|"} / node_filesystem_files{fstype!~"erofs|"} * 100 < 10
    for: 5m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
      expr: node_filesystem_files_free{fstype!~"erofs|"} / node_filesystem_files{fstype!~"erofs|"} * 100 < 10
    annotations:
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }} disk inode usage more than 90%, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.1laamqn6yrdq
      value: '{{ $value }}'
      summary: "Host disk inode usage is more than 90% (instance {{ $labels.instance }})"
  - alert: NodeHighMemoryUsage
    expr: label_replace((((node_memory_MemTotal_bytes-node_memory_MemFree_bytes-node_memory_Cached_bytes)/(node_memory_MemTotal_bytes)*100) >= 85) and (node_memory_MemFree_bytes+node_memory_Cached_bytes < 8*1000*1000*1000), "node","$1","instance", "(.+)") * on (cluster, node) group_right() kube_node_labels{label_eks_amazonaws_com_capacity_type!="SPOT"}
    for: 10m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
      expr: label_replace((((node_memory_MemTotal_bytes-node_memory_MemFree_bytes-node_memory_Cached_bytes)/(node_memory_MemTotal_bytes)*100) >= 85) and (node_memory_MemFree_bytes+node_memory_Cached_bytes < 8*1000*1000*1000), "node","$1","instance", "(.+)") * on (cluster, node) group_right() kube_node_labels{label_eks_amazonaws_com_capacity_type!="SPOT"}
    annotations:
      description: 'instance: {{ $labels.node }} memory usage more than 85%, values:{{ $value }}'
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.rcqb2vugl38f
      value: '{{ $value }}'
      summary: "Host memory usage is more than 85% (node {{ $labels.node }})"
  - alert: NodeHighCPUUsage
    expr: 100 - (avg by(instance,cluster) (irate(node_cpu_seconds_total{mode="idle", provider_type!~"aws-free-tier|alicloud-serverless"}[5m])) * 100) > 85
    for: 15m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: warning
      expr: 100 - (avg by(instance,cluster) (irate(node_cpu_seconds_total{mode="idle", provider_type!~"aws-free-tier|alicloud-serverless"}[5m])) * 100) > 85
    annotations:
      summary: "Host high CPU load (instance {{ $labels.instance }})"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.5l5xttfiy56o
      value: '{{ $value }}'
      description: "CPU load is > 85%\n  VALUE = {{ $value }}"
  - alert: NodeDiskIsReadonly
    expr: node_filesystem_readonly{cluster=~".*shoot.*", fstype=~"(ext.|xfs)"} == 1
    for: 5m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: critical
      response_class: page
      expr: node_filesystem_readonly{cluster=~".*shoot.*", fstype=~"(ext.|xfs)"} == 1
    annotations:
      summary: "Node disk is readonly (instance {{ $labels.instance }})"
      runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.taxo9pqbozor
      value: '{{ $value }}'
      description: 'instance: {{ $labels.instance }}, device: {{ $labels.device }}, mountpoint: {{$labels.mountpoint}}, values:{{ $value }}'
  - alert: RaftVolumeIOPSTooHigh
    expr: |
      (
        rate(aws_ebs_csi_read_ops_total[1m])
        + ignoring(operation)
        rate(aws_ebs_csi_write_ops_total[1m])
      )
      * on(volume_id) group_left(persistentvolumeclaim) (
        label_replace(
          kube_persistentvolume_info{csi_driver="ebs.csi.aws.com"},
          "volume_id", "$1", "csi_volume_handle", "(.+)"
        )
        * on(persistentvolume) group_left(persistentvolumeclaim)
        label_replace(
          kube_persistentvolumeclaim_info{persistentvolumeclaim=~"tikv-raftlog-.*"},
          "persistentvolume", "$1", "volumename", "(.+)"
        )
      ) > 2500
    for: 5m
    labels:
      component: node
      oncall_team: dedicated_service
      severity: major
      response_class: ticket
    annotations:
      summary: "Raft volume IOPS too high (volume {{ $labels.k8persistentvolumeclaim }} {{ $labels.volume_id }})"
      value: '{{ $value }}'
      description: "Raft volume IOPS is too high\n  VALUE = {{ $value }}"
  - alert: TikvDataVolumeIOPSTooHigh
    expr: |
      max(
        label_replace((
        ( # get IOPS usage with `namespace`, `persistentvolume` and `persistentvolumeclaim` labels
          (
            label_replace(
              kube_persistentvolumeclaim_info{persistentvolumeclaim=~"tikv-db-tikv-.*"},
              "persistentvolume", "$1", "volumename", "(.+)"
            )
          )
          * on (persistentvolume) group_left()
          ( # get the sum of read + write IOPS usage
            (
              rate(aws_ebs_csi_read_ops_total[1m])
              + ignoring(operation)
              rate(aws_ebs_csi_write_ops_total[1m])
            )
            * on (volume_id) group_right()
            (
              label_replace(
                kube_persistentvolume_info{csi_driver="ebs.csi.aws.com"},
                "volume_id", "$1", "csi_volume_handle", "(.+)"
              )
            )
          )
        )
        / on (tenant, namespace, persistentvolume,persistentvolumeclaim)
        label_replace(
          (
            ( # Calculate the max IOPS usage based on the storage size,  iops = sizeInGB*4, max iops is 16000, min iops is 3000
              clamp_min(
                clamp_max(
                  kube_persistentvolumeclaim_resource_requests_storage_bytes{persistentvolumeclaim=~"tikv-db-tikv-.*"}/1024/1024/1024*4,
                  16000
                ),
                3000
              )
            )
            * on (namespace,persistentvolumeclaim) group_right()
            kube_persistentvolumeclaim_info{persistentvolumeclaim=~"tikv-db-tikv-.*"}
          ),
          "persistentvolume", "$1", "volumename", "(.+)"
        )
        ), "cluster_id", "$1", "namespace", "tidb(.+)")
      ) by (tenant, cluster_id) *100 > 80
    for: 5m
    labels:
      component: resilience
      oncall_team: resilience
      severity: major
      response_class: ticket
    annotations:
      summary: Tikv Data volume IOPS too high than 80%, tenant {{ $labels.tenant }}, cluster {{ $labels.cluster_id }}
      value: '{{ printf "%.2f" $value }}%'
  - alert: EBSVolumeIOPSExceeded
    expr: rate(aws_ebs_csi_exceeded_iops_seconds_total[2m]) > 0.5
    for: 2m
    labels:
      component: resilience
      oncall_team: resilience
      severity: warning
    annotations:
      summary: EBS volume IOPS throttled ({{ $value | humanizePercentage }} of time) on instance {{ $labels.instance }}. Please evaluate whether EBS IOPS scaling is needed and contact the customer via TAM if necessary.
  - alert: EBSVolumeThroughputExceeded
    expr: rate(aws_ebs_csi_exceeded_tp_seconds_total[2m]) > 0.1
    for: 5m
    labels:
      component: resilience
      oncall_team: resilience
      severity: warning
    annotations:
      summary: EBS volume throughput throttled ({{ $value | humanizePercentage }} of time) on instance {{ $labels.instance }}. Please evaluate whether EBS throughput scaling is needed and contact the customer via TAM if necessary.