groups:
  - name: swat-kernel-japan-control-plane
    rules:
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: UnhealthyTiDBComponentPodForJapan
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 20 minutes.
        expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed", namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152", pod=~"db-(pd|tidb|tikv|tiflash|tiproxy).*",tenant=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1372813089191061283|1372813089209283835|1372813089209272332"} == 1
        for: 20m
        labels:
          component: kubernetes
          oncall_team: dedicated_service
          severity: critical
          response_class: page
          type: shoot
          tier: dedicated
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3492
          regions: japan
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiKVOOMRelatedPodRestartsForJapan
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
                pod=~"db-tikv-.*",
                namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
              pod=~"db-tikv-.*",
              namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
          regions: japan
        annotations:
          description: >
            TiKV Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted, container last terminated reason: {{ $labels.reason }}.
            source_tcoc: TCOC-4157
          summary: TiKV pod restarted after an OOMKill
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiDBOOMRelatedPodRestartsForJapan
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
                pod=~"db-tidb-.*",
                namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
              pod=~"db-tidb-.*",
              namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tidb-executor
          oncall_team: tidb_executor
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
          regions: japan
        annotations:
          message: >
            TiDB Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiFlashOOMRelatedPodRestartsForJapan
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
                pod=~"db-tiflash-.*",
                namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
              pod=~"db-tiflash-.*",
              namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
          regions: japan
        annotations:
          message: >
            TiFlash Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiCDCOOMRelatedPodRestartsForJapan
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
                pod=~"db-ticdc-.*",
                namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
                pod=~"db-ticdc-.*",
                namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152",
                reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: ticdc-kernel
          oncall_team: ticdc
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
          regions: japan
        annotations:
          message: >
            TiCDC Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: PDOOMRelatedPodRestartsForJapan
        expr: |
          sum by (namespace, pod) (
            increase(
              kube_pod_container_status_restarts_total{
                tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
                pod=~"db-pd-.*",
                namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152"
              }[15m]
            ) > 0
          )
          * on(namespace, pod) group_left(reason)
          max by (namespace, pod, reason) (
            kube_pod_container_status_last_terminated_reason{
              tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
              pod=~"db-pd-.*",
              namespace=~"tidb1379661944646415973|tidb10852297103751070537|tidb10602564149339879488|tidb10123293495242577103|tidb1379661944646229300|tidb10123293495242577103|tidb1379661944646229300|tidb1379661944646413522|tidb1379661944638454106|tidb1379661944636084123|tidb1379661944646414117|tidb10947714666900611238|tidb1379661944646416287|tidb1379661944646416286|tidb10244988415975579886|tidb10143988032832670331|tidb10700533136499556764|tidb10766841556523235152",
              reason="OOMKilled"
            } > 0
          )
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: pd
          oncall_team: pd
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4157
          regions: japan
        annotations:
          message: >
            PD Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled.
            source_tcoc: TCOC-4157