groups:
  - name: dataplane-components.rules
    rules:
      - alert: UnhealthyPod
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 10 minutes.
        expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed", namespace=~"tidb-admin|kube-system|cert-manager", pod!~"dataflow-agent.*|tiflow.*"} == 1
        for: 10m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: critical
          response_class: page
          type: shoot
          tier: dedicated
      - alert: UnhealthyPod
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 10 minutes.
        expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed", pod=~"cluster-agent-.*"} == 1
        for: 10m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: major
          response_class: ticket
          type: shoot
          tier: dedicated
      - alert: UnhealthyTiDBComponentPod
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 15 minutes.
        expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed", namespace=~"tidb\\d+", pod=~"db-(pd|tidb|tikv|tiflash|tiproxy).*", tenant!~"1372813089209238420|1372813089209061633|1372813089209289635"} == 1
        for: 20m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: critical
          response_class: page
          type: shoot
          tier: dedicated
      - alert: NodeNotReady
        expr: |
          (
            kube_node_status_condition{condition="Ready",status="true"} * on(node,k8s_cluster_info) group_right() kube_node_labels{label_eks_amazonaws_com_nodegroup!~".*-o11y.*"}
          ) * on(node,k8s_cluster_info) group_left(provider_id) kube_node_info == 0
        for: 15m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: major
          response_class: ticket
          type: shoot
          tier: dedicated
        annotations:
          description: "Node {{ $labels.node }} in {{ $labels.cluster }} has been unready for 15m+ \n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: NodeNotReady-O11Y
        expr: |
          (
            kube_node_status_condition{condition="Ready",status="true"} * on(node,k8s_cluster_info) group_right() kube_node_labels{label_eks_amazonaws_com_nodegroup=~".*-o11y.*"}
          ) * on(node,k8s_cluster_info) group_left(provider_id) kube_node_info == 0
        for: 15m
        labels:
          component: o11y
          oncall_team: o11y
          severity: major
          response_class: ticket
          type: shoot
          tier: dedicated
        annotations:
          description: "Node {{ $labels.node }} in {{ $labels.cluster }} has been unready for 15m+ \n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: NodeNotReadyForLongTime
        expr: |
          (
            kube_node_status_condition{condition="Ready",status="true"} * on(node,k8s_cluster_info) group_right() kube_node_labels{label_eks_amazonaws_com_nodegroup!~".*-o11y.*"}
          ) * on(node,k8s_cluster_info) group_left(provider_id) kube_node_info == 0
        for: 50m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: critical
          response_class: page
          type: shoot
          tier: dedicated
        annotations:
          description: "Node {{ $labels.node }} in {{ $labels.cluster }} has been unready for 50m+ \n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
      - alert: NodeNotReadyForLongTime-O11Y
        expr: |
          (
            kube_node_status_condition{condition="Ready",status="true"} * on(node,k8s_cluster_info) group_right() kube_node_labels{label_eks_amazonaws_com_nodegroup=~".*-o11y.*"}
          ) * on(node,k8s_cluster_info) group_left(provider_id) kube_node_info == 0
        for: 50m
        labels:
          component: o11y
          oncall_team: o11y
          severity: critical
          response_class: page
          type: shoot
          tier: dedicated
        annotations:
          description: "Node {{ $labels.node }} in {{ $labels.cluster }} has been unready for 50m+ \n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

      - alert: CoreDNSLatencyHigh
        annotations:
          description: CoreDNS in {{ $labels.cluster }} has 99th percentile latency of {{ $value }} seconds.
          runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednslatencyhigh
          summary: CoreDNS is experiencing high 99th percentile latency.
        expr: |
          histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job="coredns"}[5m])) without (instance,pod)) > 2
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated

      - alert: CoreDNSForwardLatencyHigh
        annotations:
          description: CoreDNS in {{ $labels.cluster }} has 99th percentile latency of {{ $value }} seconds forwarding
            requests.
          runbook_url: https://github.com/povilasv/coredns-mixin/tree/master/runbook.md#alert-name-corednsforwardlatencyhigh
          summary: CoreDNS is experiencing high latency forwarding requests.
        expr: |
          histogram_quantile(0.99, sum(rate(coredns_forward_request_duration_seconds_bucket{job="coredns"}[5m])) without (pod, instance, rcode)) > 4
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated

      - alert: ENA_Bandwidth_In_Allowance_Exceeded
        expr: |
            label_replace(label_replace(
              rate(node_ethtool_bw_in_allowance_exceeded{}[2m]),
              "tenant_id", "$1", "tenant", "(.*)"
            ), "node", "$1", "instance", "(.*)")
            * on(node,k8s_cluster_info)
              group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
              kube_node_labels{}
            > 100
        for: 1h
        labels:
          severity: warning
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          # visibility: region-tse
        annotations:
          description: 'Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} bandwidth-in exceeded. Instance {{ $labels.instance }} from {{ $labels.cluster }} has a bandwidth in allowwance greater than {{ $value }}'
          summary: 'EC2 inbound bandwidth allowance exceeded for 1 hour'
          runbook_url: https://fx460onx10s.feishu.cn/wiki/Krjsw8orEilio1kfVPmcPvh7ncg?table=tblbt2tB67rjtzMr&view=vewisBm5UP#doxcnMncYOYLCPOVVy8h0YEIarh

      - alert: ENA_Bandwidth_Out_Allowance_Exceeded
        expr: |
          label_replace(label_replace(
            rate(node_ethtool_bw_out_allowance_exceeded{}[2m]),
            "tenant_id", "$1", "tenant", "(.*)"
          ), "node", "$1", "instance", "(.*)")
          * on(node,k8s_cluster_info)
            group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
            kube_node_labels{}
          > 100
        for: 1h
        labels:
          severity: warning
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          # visibility: region-tse
        annotations:
          description: 'Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} bandwith-out traffic exceeded. value: {{ $value }}'
          summary: 'EC2 outbound bandwidth allowance exceeded for 1 hour'
          runbook_url: https://fx460onx10s.feishu.cn/wiki/Krjsw8orEilio1kfVPmcPvh7ncg?table=tblbt2tB67rjtzMr&view=vewisBm5UP#doxcniakPMvzDiQkI4yrKRvbVX6

      - alert: ENA_PPS_Allowance_Exceeded
        expr: |
            label_replace(label_replace(
              rate(node_ethtool_pps_allowance_exceeded{}[2m]),
              "tenant_id", "$1", "tenant", "(.*)"
            ), "node", "$1", "instance", "(.*)")
            * on(node,k8s_cluster_info)
              group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
              kube_node_labels{label_eks_amazonaws_com_nodegroup!~".*-o11y.*"}
            > 0
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          visibility: region-tse
        annotations:
          description: 'Packet Per Second (PPS) allowance for instance {{ $labels.instance }} in {{ $labels.cluster }} is greater than 0. pps_allowance_exceeded indicates number of packets queued or dropped due to Packet Per Second (PPS) exceeding the allowance for the instance. PPS allowance is enforced separately to the overall bandwidth allowance and, while the instance may still be under overall bandwidth allowance, the PPS allowance may exceed if the mean packet size is small. device: {{ $labels.device }}'
          summary: 'EC2 packet-per-second allowance exceeded for 30 minutes'
          runbook_url: https://fx460onx10s.feishu.cn/wiki/Krjsw8orEilio1kfVPmcPvh7ncg?table=tblbt2tB67rjtzMr&view=vewisBm5UP#doxcnHVR8yeGECTW9WbU1Qsoy8d
      - alert: ENA_PPS_Allowance_Exceeded-O11Y
        expr: |
            label_replace(label_replace(
              rate(node_ethtool_pps_allowance_exceeded{}[5m]),
              "tenant_id", "$1", "tenant", "(.*)"
            ), "node", "$1", "instance", "(.*)")
            * on(node,k8s_cluster_info)
              group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
              kube_node_labels{label_eks_amazonaws_com_nodegroup=~".*-o11y.*"}
            > 10
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: o11y
          oncall_team: o11y
          type: shoot
          tier: dedicated
          visibility: region-tse
        annotations:
          description: 'Packet Per Second (PPS) allowance for instance {{ $labels.instance }} in {{ $labels.cluster }} is greater than 0. pps_allowance_exceeded indicates number of packets queued or dropped due to Packet Per Second (PPS) exceeding the allowance for the instance. PPS allowance is enforced separately to the overall bandwidth allowance and, while the instance may still be under overall bandwidth allowance, the PPS allowance may exceed if the mean packet size is small. device: {{ $labels.device }}'
          summary: 'EC2 packet-per-second allowance exceeded for 30 minutes'
          runbook_url: https://fx460onx10s.feishu.cn/wiki/Krjsw8orEilio1kfVPmcPvh7ncg?table=tblbt2tB67rjtzMr&view=vewisBm5UP#doxcnHVR8yeGECTW9WbU1Qsoy8d

      - alert: ENA_Conntrack_Allowance_Exceeded
        expr: |
            label_replace(label_replace(
              rate(node_ethtool_conntrack_allowance_exceeded{}[2m]),
              "tenant_id", "$1", "tenant", "(.*)"
            ), "node", "$1", "instance", "(.*)")
            * on(node,k8s_cluster_info)
              group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
              kube_node_labels{}
            > 0
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          visibility: region-tse
        annotations:
          description: 'conntrack_allowance_exceeded indicates the number of packets dropped due to exhaustion of tracked session allowance for the instance, new sessions will fail to establish once this allowance is exceeded. cluster: {{ $labels.cluster }}, device: {{ $labels.device }}, instance: {{ $labels.instance }}'
          summary: 'EC2 connection tracking allowance exceeded for 30 minutes'
          runbook_url: https://pingcap.feishu.cn/wiki/LCjYwa3gFiYvmdkNqG6cY7VenYd#DKXjdAJxUoMSS7x0KC0cyNBCnlb

      - alert: ENA_Linklocal_Allowance_Exceeded
        expr: |
            label_replace(label_replace(
              rate(node_ethtool_linklocal_allowance_exceeded{}[2m]),
              "tenant_id", "$1", "tenant", "(.*)"
            ), "node", "$1", "instance", "(.*)")
            * on(node,k8s_cluster_info)
              group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
              kube_node_labels{label_eks_amazonaws_com_nodegroup!~".*-o11y.*"}
            > 0
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          visibility: region-tse
        annotations:
          description: 'linklocal_allowance_exceeded indicates the number of packets dropped due to PPS rate allowance exceeded for local services such as Route 53 DNS Resolver, Instance Metadata Service, Amazon Time Sync Service. This often points to suboptimal design choices or misconfiguration. This allowance is the same across instances. cluster: {{ $labels.cluster }}, device: {{ $labels.device }}, instance: {{ $labels.instance }}'
          summary: 'EC2 link-local service PPS allowance exceeded for 30 minutes'
          runbook_url: https://pingcap.feishu.cn/wiki/LCjYwa3gFiYvmdkNqG6cY7VenYd#DKXjdAJxUoMSS7x0KC0cyNBCnlb

      - alert: ENA_Linklocal_Allowance_Exceeded-O11Y
        expr: |
            label_replace(label_replace(
              rate(node_ethtool_linklocal_allowance_exceeded{}[2m]),
              "tenant_id", "$1", "tenant", "(.*)"
            ), "node", "$1", "instance", "(.*)")
            * on(node,k8s_cluster_info)
              group_left(label_node_kubernetes_io_instance_type, label_tidb_cluster,label_cluster, label_tidb_component,label_component,label_eks_amazonaws_com_nodegroup,label_topology_kubernetes_io_zone,label_usedby)
              kube_node_labels{label_eks_amazonaws_com_nodegroup=~".*-o11y.*"}
            > 0
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: o11y
          oncall_team: o11y
          type: shoot
          tier: dedicated
          visibility: region-tse
        annotations:
          description: 'linklocal_allowance_exceeded indicates the number of packets dropped due to PPS rate allowance exceeded for local services such as Route 53 DNS Resolver, Instance Metadata Service, Amazon Time Sync Service. This often points to suboptimal design choices or misconfiguration. This allowance is the same across instances. cluster: {{ $labels.cluster }}, device: {{ $labels.device }}, instance: {{ $labels.instance }}'
          summary: 'EC2 link-local service PPS allowance exceeded for 30 minutes'
          runbook_url: https://pingcap.feishu.cn/wiki/LCjYwa3gFiYvmdkNqG6cY7VenYd#DKXjdAJxUoMSS7x0KC0cyNBCnlb

      - alert: NodeHighNumberConntrackEntriesUsed
        annotations:
          description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused/
          summary: Number of conntrack are getting close to the limit.
        expr: |
          label_replace(
            (node_nf_conntrack_entries / node_nf_conntrack_entries_limit),
            "tenant_id", "$1", "tenant", "(.*)"
          ) > 0.7
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          visibility: region-tse

      - alert: NodeHighNumberConntrackEntriesUsedCritical
        annotations:
          description: '{{ $value | humanizePercentage }} of conntrack entries are used.'
          runbook_url: https://runbooks.prometheus-operator.dev/runbooks/node/nodehighnumberconntrackentriesused/
          summary: Number of conntrack are getting close to the limit.
        expr: |
          label_replace(
            (node_nf_conntrack_entries / node_nf_conntrack_entries_limit),
            "tenant_id", "$1", "tenant", "(.*)"
          ) > 0.95
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: kubernetes
          oncall_team: dedicated_service
          type: shoot
          tier: dedicated
          visibility: region-tse