groups:
  - name: swat-kernel-tcocp-control-plane
    rules:
      - alert: ImportFailForBig3
        annotations:
          message: >
            import {{ $labels.import_id }} for cluster {{ $labels.cluster_id }} failed: {{ $labels.msg }}.
            source_tcoc: TCOC-3634
        expr: |
          dataflow_import_fail{tenant=~"1372813089209238420|1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: lightning
          oncall_team: tidb_x_tools
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3634
      - alert: LighningPodNotReadyForBig3
        annotations:
          summary: 'Lightning Pod is not ready for 15 min'
          message: Lightning of Tenant {{ $labels.tenant}}, Cluster {{ $labels.namespace }} has been in a non-ready state for longer than 15 minutes.
        expr: |
          sum(kube_pod_status_phase{job="kube-state-metrics", phase=~"Pending|Unknown", pod=~"import.*",tenant=~"1372813089209238420|1372813089209061633|.*1372813089209289635", cluster!~".*1372813089454536384.*|.*1372813089454561820.*|.*1372813089454576822.*|.*1372813089454574660.*"}) by (namespace, tenant) > 0
        for: 15m
        labels:
          component: lightning
          oncall_team: tidb_x_tools
          severity: critical
          response_class: page
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3817
      - alert: TiCDCPodsUnhealthyPhaseForBig3
        expr: kube_pod_status_phase{phase!="Running", pod=~".*-ticdc-.*",tenant=~"1372813089209238420|1372813089209061633|.*1372813089209289635", cluster!~".*1372813089454536384.*|.*1372813089454561820.*|.*1372813089454576822.*|.*1372813089454574660.*"} == 1
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: ticdc
          oncall_team: tidb_x_tools
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3799
        annotations:
          message: >
            TiCDC pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} 
            has been in {{ $labels.phase }} phase for longer than 15 minutes    
      - alert: UnhealthyTiDBComponentPodForBig3
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 20 minutes.
        expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed", namespace=~"tidb\\d+", pod=~"db-(pd|tidb|tikv|tiflash|tiproxy).*",tenant=~"1372813089209238420|1372813089209061633|.*1372813089209289635", cluster!~".*1372813089454536384.*|.*1372813089454561820.*|.*1372813089454576822.*|.*1372813089454574660.*"} == 1
        for: 20m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: critical
          response_class: page
          type: shoot
          tier: dedicated
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3492
      - alert: EBSTiKVIOPSExceededForBig3
        expr: |-
          max by(cluster_id)(
            label_replace(
              max by(namespace)(
                label_replace(
                  rate(aws_ebs_csi_exceeded_iops_seconds_total{
                    tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054"
                  }[2m]),
                  "node", "$1", "instance", "(.+)"
                )
                * on(node) group_right()
                kube_pod_info{pod=~"db-tikv-.*"}
              ) > 0.5,
              "cluster_id", "$1", "namespace", "tidb(.*)"
            )
          )
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3905
        annotations:
          summary: TiKV EBS IOPS throttled on cluster {{ $labels.cluster_id }}.
          message: >
            TiKV EBS IOPS throttled ({{ $value | humanizePercentage }} of time) on cluster {{ $labels.cluster_id }}.
            Please evaluate whether EBS IOPS scaling is needed and contact the customer via TAM if necessary.
            source_tcoc: TCOC-3905
      - alert: EBSTiKVThroughputExceededForBig3
        expr: |-
          max by(cluster_id)(
            label_replace(
              max by(namespace)(
                label_replace(
                  rate(aws_ebs_csi_exceeded_tp_seconds_total{
                    tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054"
                  }[2m]),
                  "node", "$1", "instance", "(.+)"
                )
                * on(node) group_right()
                kube_pod_info{pod=~"db-tikv-.*"}
              ) > 0.1,
              "cluster_id", "$1", "namespace", "tidb(.*)"
            )
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3905
        annotations:
          summary: TiKV EBS throughput throttled on cluster {{ $labels.cluster_id }}.
          message: >
            TiKV EBS throughput throttled ({{ $value | humanizePercentage }} of time) on cluster {{ $labels.cluster_id }}.
            Please evaluate whether EBS throughput scaling is needed and contact the customer via TAM if necessary.
            source_tcoc: TCOC-3905
      - alert: TiKVOOMRelatedPodRestartsForBig3
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
                project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                pod=~"db-tikv-.*"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
              project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
              pod=~"db-tikv-.*",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
        annotations:
          description: >
            TiKV Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted, container last terminated reason: {{ $labels.reason }}.
            source_tcoc: TCOC-4157
          summary: TiKV pod restarted after an OOMKill
      - alert: TiDBOOMRelatedPodRestartsForBig3
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
                project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                pod=~"db-tidb-.*"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
              project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
              pod=~"db-tidb-.*",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tidb-executor
          oncall_team: tidb_executor
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
        annotations:
          message: >
            TiDB Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      - alert: TiFlashOOMRelatedPodRestartsForBig3
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
                project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                pod=~"db-tiflash-.*"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
              project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
              pod=~"db-tiflash-.*",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
        annotations:
          message: >
            TiFlash Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      - alert: TiCDCOOMRelatedPodRestartsForBig3
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
                project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                pod=~"db-ticdc-.*"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
              project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
              pod=~"db-ticdc-.*",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: ticdc-kernel
          oncall_team: ticdc
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
        annotations:
          message: >
            TiCDC Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      - alert: PDOOMRelatedPodRestartsForBig3
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
                project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                pod=~"db-pd-.*"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
              project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
              pod=~"db-pd-.*",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: pd
          oncall_team: pd
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
        annotations:
          message: >
            PD Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      - alert: EBSTiKVVolumeIOHang
        expr: |-
          label_replace(
            (
              (aws_ebs_csi_volume_queue_length > 0)
              * on(volume_id) group_left(persistentvolume, storageclass)
              label_replace(
                max by(csi_volume_handle, persistentvolume, storageclass) (
                  kube_persistentvolume_info{csi_driver="ebs.csi.aws.com", storageclass=~".*tikv.*"}
                ),
                "volume_id", "$1", "csi_volume_handle", "(.+)"
              )
              * on(persistentvolume) group_left(persistentvolumeclaim, namespace)
              label_replace(
                max by(volumename, persistentvolumeclaim, namespace) (
                  kube_persistentvolumeclaim_info
                ),
                "persistentvolume", "$1", "volumename", "(.+)"
              )
              * on(namespace, persistentvolumeclaim) group_left(pod)
              max by(namespace, persistentvolumeclaim, pod) (
                kube_pod_spec_volumes_persistentvolumeclaims_info
              )
            )
            unless on(instance, volume_id)
            (rate(aws_ebs_csi_write_ops_total[2m]) > 0)
            unless on(instance, volume_id)
            (rate(aws_ebs_csi_read_ops_total[2m]) > 0),
            "cluster_id", "$1", "namespace", "tidb(.*)"
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
        annotations:
          summary: EBS volume IO hang on TiKV pod (IO queued but not completing) - suspected EBS failure on cluster {{ $labels.cluster_id }}.
          message: >
            Suspected EBS volume failure: queue_length={{ $value }} but no read/write IO completing for 5min on
            pod {{ $labels.namespace }}/{{ $labels.pod }}, volume {{ $labels.volume_id }}, instance {{ $labels.instance }}.
            source_tcoc: TCOC-4157
      # - alert: HighMemoryUsageOver20GBForRestorePod
      #   expr: |-
      #      label_replace(
      #      (
      #        node_memory_MemTotal_bytes{tenant=~"1372813089209061633|.*1372813089209289635" }
      #        - node_memory_MemFree_bytes{tenant=~"1372813089209061633|.*1372813089209289635" }
      #        - node_memory_Buffers_bytes{tenant=~"1372813089209061633|.*1372813089209289635" }
      #        - node_memory_Cached_bytes{tenant=~"1372813089209061633|.*1372813089209289635" }
      #      ),
      #      "node", "$1", "instance", "(.+)")
      #      * on(node) group_right() label_replace(kube_pod_info{pod=~"restore.*", tenant=~"1372813089209061633|.*1372813089209289635"},  "cluster_id", "$1", "namespace", "(.+)")
      #      > 20 * 1024 * 1024 * 1024
      #   for: 0s
      #   labels:
      #     severity: critical
      #     component: br
      #     stability_governance: kernel-governance
      #   annotations:
      #     message: >
      #       Memory usage on Pod {{ $labels.namespace }}/{{ $labels.pod }} has exceeded 20GB.
      - alert: TiKVStorageExceed80ForBig3
        expr: |
          sum(tikv_store_size_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", type="used"}) by (tidb_cluster_id) / sum (tikv_store_size_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", type="capacity"}) by (tidb_cluster_id) > 0.8
        for: 10m
        labels:
          severity: warning
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
        annotations:
          message: >
            TiFlash store space used more than 80% on pod {{ $labels.namespace }}/{{ $labels.pod }}. Need notify the customer.
#      - alert: NodeHighCPU
#        expr: |
#          (
#            100 - avg by (instance) (
#              irate(
#                node_cpu_seconds_total{
#                  mode="idle",
#                  project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
#                }[5m]
#              ) * 100
#            )
#          )
#          * on (instance)
#          group_left(namespace)
#          label_replace(
#            max by (node, namespace) (
#              kube_pod_info{
#                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054"
#              }
#            ),
#            "instance", "$1", "node", "(.*)"
#          )
#          > 90
#        for: 600s
#        labels:
#          severity: major
#          component: node-info
#          stability_governance: kernel-tcocp
#          source_tcoc: TCOC-4572
#        annotations:
#          message: >
#            Node {{ $labels.namespace }}/{{ $labels.instance }} CPU over 90% for 10min.
#            source_tcoc: TCOC-4572