groups:
- name: kubelet.rules
  rules:
  - expr: |
      histogram_quantile(0.99, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster,instance, le) * on(cluster,instance) group_left(node) kubelet_node_name{job="kubelet"})
    labels:
      quantile: "0.99"
    record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.9, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster,instance, le) * on(cluster,instance) group_left(node) kubelet_node_name{job="kubelet"})
    labels:
      quantile: "0.9"
    record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
  - expr: |
      histogram_quantile(0.5, sum(rate(kubelet_pleg_relist_duration_seconds_bucket[5m])) by (cluster,instance, le) * on(cluster,instance) group_left(node) kubelet_node_name{job="kubelet"})
    labels:
      quantile: "0.5"
    record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
- name: kube-prometheus-node-recording.rules
  rules:
  - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY
      (instance)
    record: instance:node_cpu:rate:sum
  - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}))
      by (cluster, instance)
    record: instance:node_filesystem_usage:sum
  - expr: sum(rate(node_network_receive_bytes_total[3m])) by (cluster, instance)
    record: instance:node_network_receive_bytes:rate:sum
  - expr: sum(rate(node_network_transmit_bytes_total[3m])) by (cluster, instance)
    record: instance:node_network_transmit_bytes:rate:sum
  - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT
      (cpu, mode) / on(cluster, instance) GROUP_LEFT() count(sum(node_cpu_seconds_total)
      BY (cluster, instance, cpu)) BY (cluster, instance)
    record: instance:node_cpu:ratio
  - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) by (cluster)
    record: cluster:node_cpu:sum_rate5m
  - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total)
      by (cluster, instance, cpu))
    record: cluster:node_cpu:ratio
- name: kubernetes-system-kubelet
  rules:
  - alert: KubeNodeNotReady
    annotations:
      message: '{{ $labels.node }} has been unready for more than 15 minutes.'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodenotready
    expr: |
      kube_node_status_condition{job="kube-state-metrics",condition="Ready",status="true"} == 0
    for: 15m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service
  - alert: KubeNodeUnreachable
    annotations:
      message: '{{ $labels.node }} is unreachable and some workloads may be rescheduled. region: {{ $labels.region}}, component: {{ $labels.label_serverless_tidbcloud_com_node }}, instance-type: {{ $labels.label_node_kubernetes_io_instance_type }}, capacity-type: {{ $labels.label_eks_amazonaws_com_capacity_type }}'
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodeunreachable
    expr: |
      (kube_node_spec_taint{job="kube-state-metrics",key="node.kubernetes.io/unreachable",effect="NoSchedule"} == 1) +on(node) group_left(label_serverless_tidbcloud_com_node,label_node_kubernetes_io_instance_type,label_eks_amazonaws_com_capacity_type) (0 * kube_node_labels{label_eks_amazonaws_com_capacity_type!="SPOT"})
    for: 2m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service
  # Deprecated in kube-state-metrics v2
  - alert: KubeletTooManyPods
    annotations:
      message: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage
        }} of its Pod capacity.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
    expr: |
      max(max(kubelet_running_pod_count{job="kubelet"}) by(cluster, instance) * on(cluster, instance) group_left(node) kubelet_node_name{job="kubelet"}) by(cluster, node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"} != 1) by(cluster, node) > 0.95
    for: 15m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service
  - alert: KubeletTooManyPodsV2
    annotations:
      message: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage
        }} of its Pod capacity.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
    expr: |
      max(max(kubelet_running_pod_count{job="kubelet"}) by(cluster,instance) * on(cluster,instance) group_left(node) kubelet_node_name{job="kubelet"}) by(cluster, node) / max(kube_node_status_capacity{job="kube-state-metrics", resource="pods"} != 1) by(cluster, node) > 0.95
    for: 15m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service
  - alert: KubeNodeReadinessFlapping
    annotations:
      message: The readiness status of node {{ $labels.node }} has changed {{ $value
        }} times in the last 15 minutes.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping
    expr: |
      sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (cluster, node) > 2
    for: 15m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service
  - alert: KubeletPlegDurationHigh
    annotations:
      message: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration
        of {{ $value }} seconds on node {{ $labels.node }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh
    expr: |
      node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
    for: 5m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service
  - alert: KubeletPodStartUpLatencyHigh
    annotations:
      message: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds
        on node {{ $labels.node }}.
      runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh
    expr: |
      histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet"}[5m])) by (cluster,instance, le)) * on(cluster,instance) group_left(node) kubelet_node_name{job="kubelet"} > 60
    for: 15m
    labels:
      severity: warning
      component: kubernetes
      oncall_team: dedicated_service