groups:
  - name: swat-kernel-japan
    rules:
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: HighCPUImbalanceOnTiDBForJapan
        expr: |
          (
            (
              max by (cluster_id, cluster) (
                (
                  100 *                   sum(rate(node_cpu_seconds_total{
                    mode!="idle",
                    component="tidb",
                    tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
                  }[5m])) by (cluster_id, cluster, instance)
                )
                /
                (
                  sum(rate(node_cpu_seconds_total{
                    component="tidb",
                    tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
                  }[5m])) by (cluster_id, cluster, instance)
                )
              )
              /
              (
                (
                  100 * avg(
                    sum(rate(node_cpu_seconds_total{
                      mode!="idle",
                      component="tidb",
                      tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
                    }[5m])) by (cluster_id, cluster)
                  )
                )
                /
                (
                  avg(
                    sum(rate(node_cpu_seconds_total{
                      component="tidb",
                      tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
                    }[5m])) by (cluster_id, cluster)
                  )
                )
              )
            ) > 1.5
          )
          and
          (
            (
              max by (cluster_id, cluster) (
                (
                  100 * sum(rate(node_cpu_seconds_total{
                    mode!="idle",
                    component="tidb",
                    tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
                  }[5m])) by (cluster_id, cluster, instance)
                )
                /
                (
                  sum(rate(node_cpu_seconds_total{
                    component="tidb",
                    tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
                  }[5m])) by (cluster_id, cluster, instance)
                )
              )
            ) > 80
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-governance
          regions: japan
        annotations:
          summary: "TiDB instance CPU usage is significantly higher than cluster average"
          description: "One TiDB node's CPU usage exceeds 80% and is 1.5x higher than the cluster's 15-minute average. This may indicate workload imbalance, SQL hotspots, or connection skew. Please check whether it is ddl leader / analyze owner / gc owner node."
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiKVCPUImbalanceHighFor5mForJapan
        expr: |
          ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{
            mode!="idle",
            component="tikv",
            tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
          }[5m])) by (cluster_id,instance))
          /
          (sum(rate(node_cpu_seconds_total{
            component="tikv",
            tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
          }[5m])) by (cluster_id,instance)) ))
          /
          ((100 * avg(sum(rate(node_cpu_seconds_total{
            mode!="idle",
            component="tikv",
            tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
          }[5m])) by (cluster_id))
          /
          avg(sum(rate(node_cpu_seconds_total{
            component="tikv",
            tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
          }[5m])) by (cluster_id))))) > 1.5
           and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{
            mode!="idle",
            component="tikv",
            tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
          }[5m])) by (cluster_id,instance))
          /
          (sum(rate(node_cpu_seconds_total{
            component="tikv",
            tenant=~"1372813089201471287|1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"
          }[5m])) by (cluster_id,instance)) ))>65
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-governance
          regions: japan
        annotations:
          summary: "TiKV instance CPU usage is significantly higher than cluster average"
          description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV nodes continuously for 5 minutes. One TiKV node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiKVCPUImbalanceHighFor20mForJapan
        expr: |
          ((Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{
            mode!="idle",
            component="tikv",
            tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          }[5m])) by (cluster_id,instance))
          /
          (sum(rate(node_cpu_seconds_total{
            component="tikv",
            tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          }[5m])) by (cluster_id,instance)) ))
          /
          ((100 * avg(sum(rate(node_cpu_seconds_total{
            mode!="idle",
            component="tikv",
            tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          }[5m])) by (cluster_id))
          /
          avg(sum(rate(node_cpu_seconds_total{
            component="tikv",
            tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          }[5m])) by (cluster_id))))) > 1.5
           and (Max by (cluster_id)((100 * sum(rate(node_cpu_seconds_total{
            mode!="idle",
            component="tikv",
            tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          }[5m])) by (cluster_id,instance))
          /
          (sum(rate(node_cpu_seconds_total{
            component="tikv",
            tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          }[5m])) by (cluster_id,instance)) ))>85
        for: 20m
        labels:
          severity: critical
          response_class: page
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-governance
          regions: japan
        annotations:
          summary: "TiKV instance CPU usage is significantly higher than cluster average"
          description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV nodes continuously for 20 minutes. One TiKV node''s CPU usage exceeds 85% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'

      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiCDCDowntimeOver30MinForJapan
        expr: |
          up{
            component=~"ticdc",
            job=~".*ticdc",
            tenant=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1372813089191061283|1372813089209283835|1372813089209272332",
            cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"
          } == 0
        for: 30m
        labels:
          tier: dedicated
          severity: critical
          response_class: page
          component: ticdc
          oncall_team: tidb_x_tools
          stability_governance: kernel-governance
          regions: japan
        annotations:
          description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.'
          summary: '{{ $labels.component }} server has been down for over 30 minutes'
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiKVCoreMemoryUsageOverAvgTotalForJapan
        annotations:
          description: 'TiKV single node memory utilization over (Total TiKV node memory utilization + 5). value: {{ $value }} source_tcoc: TCOC-2913'
          summary: 'TiKV single node memory utilization exceeds the cluster average by 5 points'
        expr: |-
          (100 * (1 - (
              node_memory_MemAvailable_bytes{component="tikv", tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}
              /
              node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}
            ))) > (100 * (1 - avg by (tidb_cluster_id) (
            (
              node_memory_MemFree_bytes{component="tikv", tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283", cluster_id=~".*1379661944646413949|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10700533136499556764|.*10766841556523235152"} +
              node_memory_Buffers_bytes{component="tikv", tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283", cluster_id=~".*1379661944646413949|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10700533136499556764|.*10766841556523235152"} +
              node_memory_Cached_bytes{component="tikv", tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283", cluster_id=~".*1379661944646413949|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10700533136499556764|.*10766841556523235152"}
            )
            /
            node_memory_MemTotal_bytes{component="tikv", tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}
          ))+5)
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2913
          regions: japan
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiDBP95LatencySurgeForJapan
        expr: |
          histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"}[1m])) by (le, cluster_id))
          >
          5 * histogram_quantile(0.95, avg_over_time(
            sum by (le, cluster_id) (
              rate(tidb_server_handle_query_duration_seconds_bucket{component="tidb",tenant=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"}[5m])
            )[24h:5m]
          ))
        for: 5m
        labels:
          tier: dedicated
          severity: critical
          response_class: page
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
          regions: japan
        annotations:
          description: 'TiDB query P95 latency within the past 5 minutes exceeds 5 times the 24-hour P95 baseline. This may indicate query slowness, hotspots, resource contention, or downstream degradation. SOP: https://pingcap.feishu.cn/wiki/CbEqwrDpaivA0bkxd5xcmpaQnDb. cluster: {{ $labels.cluster_id }}, current 5-minutes P95 latency is {{ $value }}s, which exceeds 5x of the 24h P95 baseline'
          summary: TiDB query P95 latency significantly increased in short term
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: RequestErrorRateOver10ForJapan
        expr: |-
          (
            sum(increase(tidb_server_execute_error_total{tenant=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152", type !~ "tikv:3572"}[5m])) by (cluster_id)
            /
            sum(increase(tidb_executor_statement_total{tenant=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152", type !~ "tikv:3572"}[5m])) by (cluster_id)
          ) > 0.1
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: tidb-optimizer
          oncall_team: tidb_optimizer
          stability_governance: kernel-governance
          regions: japan
        annotations:
          message: >
            Cluster {{ $labels.cluster_id }}  the error rate is above 10%. Please check the recent errors and overall statement execution status for this cluster.
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: RapidErrorRateIncreaseOver450ForJapan
        expr: |-
          (
            avg_over_time(
              (
                sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300", type="unknown"}[10m])) by (cluster_id)
                /
                sum(increase(tidb_executor_statement_total{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}[10m])) by (cluster_id)
              )[10m:1m]
            )
            /
            avg_over_time(
              (
                sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300", type="unknown"}[1d])) by (cluster_id)
                /
                sum(increase(tidb_executor_statement_total{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}[1d])) by (cluster_id)
              )[1d:1h]
            )
          ) > 4.5
          and
          sum(increase(tidb_server_execute_error_total{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300", type="unknown"}[10m])) by (cluster_id) > 1000
        for: 0s
        labels:
          severity: major
          response_class: ticket
          component: tidb-optimizer
          stability_governance: kernel-governance
          regions: japan
          assign: kaiwen.shen@pingcap.com
        annotations:
          message: >
            Cluster {{ $labels.cluster_id }} The error rate in the last 1 minute is more than 4.5 times higher than the 5-minute average.
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiFlashLowSpace70ForJapan
        expr: |
          (
            sum(tiflash_system_current_metric_StoreSizeAvailable{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}) by (cluster_id, tenant)
            / sum(tiflash_system_current_metric_StoreSizeCapacity{tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300"}) by (cluster_id, tenant) * 100
          )
          < on(cluster_id) group_left()
          (
            (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 100 + 10
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          tier: dedicated
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2864
          regions: japan
        annotations:
          description: 'TiFlash store space used more than 70% source_tcoc: TCOC-2864. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
          runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
          value: '{{ $value }}'
          summary: "TiFlash store space used more than 70%"
      # Dynamic threshold: (1 - low_space_ratio) * 100 + 2
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiFlashLowSpace78ForJapan
        expr: |
          (
            sum(tiflash_system_current_metric_StoreSizeAvailable{tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332", cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"}) by (cluster_id, tenant)
            / sum(tiflash_system_current_metric_StoreSizeCapacity{tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332", cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"}) by (cluster_id, tenant) * 100
          )
          < on(cluster_id) group_left()
          (
            (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 100 + 2
          )
        for: 5m
        labels:
          severity: critical
          response_class: page
          tier: dedicated
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2864
          regions: japan
        annotations:
          description: 'TiFlash store space used more than 78%. Need notify the customer. source_tcoc: TCOC-2864. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
          runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
          value: '{{ $value }}'
          summary: "TiFlash store space used more than 78%"
      # Disable the following rule for ci check
      # ignore_validations: expressionDoesNotUseRangeShorterThan
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiKVSideCDCSinkMemoryHighUsageForJapan
        expr: sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*",tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10700533136499556764|.*10766841556523235152"}) by (cluster_id, instance) > 300000000
        for: 5m
        labels:
          severity: critical
          response_class: page
          expr: sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*",tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10700533136499556764|.*10766841556523235152"}) by (cluster_id, instance) > 300000000
          component: ticdc-kernel
          oncall_team: ticdc
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3032
          regions: japan
        annotations:
          message: >
            TiDB Cluster: {{ $labels.cluster_id }}
            Instance: {{ $labels.instance }}
            Usage of CDC sink memory in TiKV side is high which default is 512MB. Currently it exceed 300MB. please refer to SOP https://pingcap.feishu.cn/wiki/OTIewy2EoiAnGCkTezmc0xgHnfc to handle this case.
            source_tcoc: TCOC-3032
      # opt-in alert: mirrored as ForBig3/ForUE; will consolidate when opt-in mechanism replaces ForXxx grouping
      - alert: TiFlashRestartFrequentlyForJapan
        expr: sum(increase(kube_pod_container_status_restarts_total{container="tiflash",tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*1379661944646414117|.*10947714666900611238|.*1379661944646416287|.*1379661944646416286|.*10244988415975579886|.*10143988032832670331|.*10700533136499556764|.*10766841556523235152"}[30m]))by(container,tenant,namespace,cluster_id)>5
        for: 10m
        labels:
          tier: dedicated
          severity: critical
          response_class: page
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3758
          regions: japan
        annotations:
          summary: Tiflash container restart frequently
          message: >
            Container {{ $labels.namespace }}/{{ $labels.container }} pod {{ $labels.pod }} is restarting {{ printf "%.2f" $value }} times in 30 minutes.
            source_tcoc: TCOC-3758
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiDBClientLatencyExceeds1sForJapan
        expr: |
          max by (cluster_id) (
            histogram_quantile(0.99,
              sum by (le, store, cluster_id) (
                rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{
                  tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300",
                  instance=~".*",
                  scope="false"
                }[1m])
              ))
          )> 1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: GTOC-7734
          regions: japan
        annotations:
          description: 'TiDB client p99 latency > 1s (instant alert) source_tcoc: GTOC-7734 grafana_dashboard: TiDB-Cluster-Clinic / KV Request / RPC Layer Latency P99. store: {{ $labels.store }}, value: {{ $value }}'
          value: '{{ $value }}'
          summary: "TiDB client p99 latency > 1s (instant alert)"
      # TODO: extract to a generic alert (no tenant filter, applies to all clusters)
      - alert: TiDBClientLatencyExceeds200msForJapan
        expr: |
          max by (cluster_id) (
            histogram_quantile(0.99,
              sum by (le, store, cluster_id) (
                rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{
                  tenant=~".*1372813089201471287|.*1372813089195441286", cluster_id=~".*10123293495242577103|.*1379661944646229300",
                  instance=~".*",
                  scope="false"
                }[1m])
              ))
          ) > 0.2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: GTOC-7734
          regions: japan
        annotations:
          description: 'TiDB client p99 latency > 200ms for 5 minute source_tcoc: GTOC-7734 grafana_dashboard: TiDB-Cluster-Clinic / KV Request / RPC Layer Latency P99. store: {{ $labels.store }}, value: {{ $value }}'
          value: '{{ $value }}'
          summary: "TiDB client p99 latency > 200ms for 5 minute"
      # Kept: Japan-unique with no Big3 equivalent yet. Replace with a QPSDroppedBy% style alert when designed
      - alert: QPSDropToZeroFor2Minutes
        expr: |
          (
            sum(rate(tidb_executor_statement_total{
              tenant=~".*1372813089201471287|.*1372813089195441286|.*1372813089190911285|.*1372813089199011282|.*1372813089191061283|.*1372813089209283835|.*1372813089209272332",cluster_id=~".*1379661944646415973|.*10852297103751070537|.*10602564149339879488|.*10123293495242577103|.*1379661944646229300|.*10123293495242577103|.*1379661944646229300|.*1379661944646413522|.*1379661944638454106|.*1379661944636084123|.*10700533136499556764|.*10766841556523235152",
              status!~"failed|success|finished|deleted",
            }[1m])) == 0
          )
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: kubernetes
          oncall_team: dedicated_service
          stability_governance: kernel-governance
          regions: japan
        annotations:
          message: "The QPS in the cluster {{ $labels.cluster_id }} has dropped to zero or the metric is missing for more than 2 minutes."
      # opt-in alert: mirrored as ForBig3; enabled for Mercari (tenant 1372813089201471287)
      # Disable the following rule for ci check
      # ignore_validations: expressionDoesNotUseRangeShorterThan
      - alert: PDLeaderChangeForJapan
        expr: |
          changes(service_member_role{
            service="PD",
            tenant=~".*1372813089201471287"
          }[15s]) > 0
          unless on(instance)
          (
            (
              sum(
                count_over_time(
                  pd_service_maxprocs{
                    tenant=~".*1372813089201471287",
                    component=~".*pd.*"
                  }[30s]
                )
              ) by (instance)
              or on(instance)
              0 * pd_service_maxprocs{
                tenant=~".*1372813089201471287",
                component=~".*pd.*"
              }
            ) <= 1
          )
        labels:
          severity: major
          response_class: ticket
          tier: dedicated
          component: pd
          oncall_team: pd
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-5783
          regions: japan
        annotations:
          description: 'pd leader changes, might cause perfermance degrade. source_tcoc: TCOC-5783. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
          summary: pd leader changes, might cause perfermance degrade.