groups:
  - name: swat-kernel-tcocp
    rules:
      # Special rules for specific customers, which will alert both cloud team and kernel team
      # one problem will have two alert rule: one for cloud team, one for kernel team
      # 1. alert for tenant: Wetech:1372813089209061633|.*1372813089209289635, BingoPlus:1372813089209238420
      # 2. filter out test project: aws-pressure-test-saas(Wetech: 1372813089454536384)
      # 3. filter cdc create in less than one hour
      - alert: BizChangefeedCheckpointHighDelay8m2KernelTeamForBig3
        expr: |
          ticdc_owner_checkpoint_ts_lag{tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", changefeed !~ "changefeed-1979092402735312896"} > 480
          and on (tenant, project, tidb_cluster_id, changefeed)
          ticdc_owner_checkpoint_ts_lag{tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", changefeed !~ "changefeed-1979092402735312896"} offset 1h
        for: 1m
        labels:
          severity: critical
          response_class: page
          expr: ticdc_owner_checkpoint_ts_lag{} > 8m
          component: ticdc-kernel
          oncall_team: ticdc
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2803
        annotations:
          message: >
            Checkpoint of {{ $labels.changefeed }} delay more than 8m.
            TiDB Cluster: {{ $labels.tidb_cluster_id }}
            Reported by {{ $labels.instance }}
            Region: {{ $labels.seed_region }}
            Infra Arch: {{ $labels.seed_provider }}
            source_tcoc: TCOC-2803
      - alert: BizChangefeedIsFailedByKernelStatus2KernelTeamForBig3
        expr: ticdc_owner_status{tenant=~".*1372813089209238420|.*1372813089209261054|.*1372813089209061633|.*1372813089209289635", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"} == 2
        labels:
          severity: critical
          response_class: page
          expr: ticdc_owner_status{} == 2
          component: ticdc-kernel
          oncall_team: ticdc
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2803
        annotations:
          message: >
            Changefeed: {{ $labels.changefeed }}
            TiDB Cluster: {{ $labels.cluster_id }}
            Instance: {{ $labels.instance }}
            source_tcoc: TCOC-2803
      # - alert: BizChangefeedIsWarningByKernelStatus2KernelTeamForBig3
      #   expr: ticdc_owner_status{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", changefeed !~ "changefeed-1979092402735312896"} == 6
      #   for: 15m
      #   labels:
      #     severity: major
      #     expr: ticdc_owner_status{} == 6
      #     component: ticdc-kernel
      #     stability_governance: kernel-tcocp
      #     source_tcoc: TCOC-2803
      #   annotations:
      #     message: >
      #       Changefeed: {{ $labels.changefeed }}
      #       TiDB Cluster: {{ $labels.cluster_id }}
      #       Instance: {{ $labels.instance }}
      #       source_tcoc: TCOC-2803
      # Dynamic threshold: (1 - low_space_ratio) * 100 + 10
      # Covers Big3 + WeTech SaaS clusters (WeTech 80 merged here after dynamic threshold)
      - alert: TiFlashLowSpace70ForBig3
        expr: |
          (
            sum(tiflash_system_current_metric_StoreSizeAvailable{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", }) by (cluster_id, tenant)
            / sum(tiflash_system_current_metric_StoreSizeCapacity{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}) by (cluster_id, tenant) * 100
          )
          < on(cluster_id) group_left()
          (
            (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 100 + 10
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          tier: dedicated
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2864
        annotations:
          description: 'TiFlash store space used more than 70% source_tcoc: TCOC-2864. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
          runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
          value: "{{ $value }}"
          summary: "TiFlash store space used more than 70%"
      # Dynamic threshold: (1 - low_space_ratio) * 100 + 2
      # Covers Big3 + WeTech SaaS clusters (WeTech 88 merged here after dynamic threshold)
      - alert: TiFlashLowSpace78ForBig3
        expr: |
          (
            sum(tiflash_system_current_metric_StoreSizeAvailable{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}) by (cluster_id, tenant)
            / sum(tiflash_system_current_metric_StoreSizeCapacity{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}) by (cluster_id, tenant) * 100
          )
          < on(cluster_id) group_left()
          (
            (1 - max(pd_config_status{type="low-space-ratio"}) by (cluster_id)) * 100 + 2
          )
        for: 5m
        labels:
          severity: critical
          response_class: page
          tier: dedicated
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-2864
        annotations:
          description: 'TiFlash store space used more than 78%. Need notify the customer. source_tcoc: TCOC-2864. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
          runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.jsri2yl8n8dp
          value: "{{ $value }}"
          summary: "TiFlash store space used more than 78%"
      # TiFlashLowSpace80ForWetechSaas and TiFlashLowSpace88ForWetechSaas deleted:
      # Merged into ForBig3 rules above. Dynamic threshold from pd_config_status
      # automatically adapts to WeTech's low-space-ratio=0.9.
      # Disable the following rule for ci check
      # ignore_validations: expressionDoesNotUseRangeShorterThan
      - alert: PDLeaderChangeForBig3
        expr: |
          changes(service_member_role{
            service="PD",
            tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"
          }[15s]) > 0
          unless on(instance)
          (
            (
              sum(
                count_over_time(
                  pd_service_maxprocs{
                    tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                    component=~".*pd.*"
                  }[30s]
                )
              ) by (instance)
              or on(instance)
              0 * pd_service_maxprocs{
                tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
                project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                component=~".*pd.*"
              }
            ) <= 1
          )
        labels:
          severity: major
          response_class: ticket
          tier: dedicated
          component: pd
          oncall_team: pd
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3517
        annotations:
          description: 'pd leader changes, might cause perfermance degrade. source_tcoc: TCOC-3517. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
          summary: pd leader changes, might cause perfermance degrade.
      - alert: PDTSOFailedForBig3
        expr: increase(pd_tso_events{type="exceeded_max_retry",tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", cluster_id!~".*1379661944646416027"}[10m]) > 0
        labels:
          severity: major
          response_class: ticket
          expr: increase(pd_tso_events{type="exceeded_max_retry",tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054|.*1369847559692509618", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", cluster_id!~".*1379661944646416027"}[10m]) > 0
          tier: dedicated
          component: pd
          oncall_team: pd
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3599
        annotations:
          description: 'pd tso failed source_tcoc: TCOC-3599. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
          summary: pd tso failed
      - alert: PDTSOFailedForWetechSaas01
        expr: rate(pd_tso_events{type="exceeded_max_retry", cluster_id=~".*1379661944646416027"}[1m]) > 0.5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          tier: dedicated
          component: pd
          oncall_team: pd
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3599
        annotations:
          description: 'pd tso failed 3 times in 10min source_tcoc: TCOC-3599. cluster: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, values: {{ $value }}'
          summary: pd tso failed 3 times in 10min
      - alert: TiKVSideCDCSinkMemoryHighUsageForBig3
        expr: max by(cluster_id)(sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*",tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}) by (cluster_id, instance) > 300000000)
        for: 5m
        labels:
          severity: critical
          response_class: page
          expr: max by(cluster_id)(sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*",tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}) by (cluster_id, instance) > 300000000)
          component: ticdc-kernel
          oncall_team: ticdc
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3032
        annotations:
          message: >
            TiDB Cluster: {{ $labels.cluster_id }}
            Instance: {{ $labels.instance }}
            Usage of CDC sink memory in TiKV side is high which default is 512MB. Currently it exceed 300MB. please refer to SOP https://pingcap.feishu.cn/wiki/OTIewy2EoiAnGCkTezmc0xgHnfc to handle this case.
            source_tcoc: TCOC-3032
      - alert: TiFlashRestartFrequentlyForBig3
        expr: sum(increase(kube_pod_container_status_restarts_total{container="tiflash",tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}[30m]))by(container,tenant,namespace,cluster_id)>5
        for: 10m
        labels:
          tier: dedicated
          severity: critical
          response_class: page
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3758
        annotations:
          summary: Tiflash container restart frequently
          message: >
            Container {{ $labels.namespace }}/{{ $labels.container }} pod {{ $labels.pod }} is restarting {{ printf "%.2f" $value }} times in 30 minutes.
            source_tcoc: TCOC-3758
      - alert: TiDBClientLatencyExceeds1s
        expr: |
          max by (cluster_id) (
            histogram_quantile(0.99,
              sum by (le, store, cluster_id) (
                rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{
                  tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054|.*1369847559692509618",
                  project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                  instance=~".*",
                  scope="false"
                }[1m])
              ))
          )> 1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: GTOC-7734
        annotations:
          description: 'TiDB client p99 latency > 1s (instant alert) source_tcoc: GTOC-7734 grafana_dashboard: TiDB-Cluster-Clinic / KV Request / RPC Layer Latency P99. store: {{ $labels.store }}, value: {{ $value }}'
          value: "{{ $value }}"
          summary: "TiDB client p99 latency > 1s (instant alert)"
      - alert: TiDBClientLatencyExceeds200ms
        expr: |
          max by (cluster_id) (
            histogram_quantile(0.99,
              sum by (le, store, cluster_id) (
                rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{
                  tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054|.*1369847559692509618",
                  project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
                  instance=~".*",
                  scope="false"
                }[1m])
              ))
          ) > 0.2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: GTOC-7734
        annotations:
          description: 'TiDB client p99 latency > 200ms for 5 minute source_tcoc: GTOC-7734 grafana_dashboard: TiDB-Cluster-Clinic / KV Request / RPC Layer Latency P99. store: {{ $labels.store }}, value: {{ $value }}'
          value: "{{ $value }}"
          summary: "TiDB client p99 latency > 200ms for 5 minute"
      - alert: TiFlashWriteThroughputTooHighForBig3
        expr: |
          sum(rate(tiflash_storage_throughput_bytes{
            job=~".*tiflash.*",
            tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
            type="write"
          }[2m])) by (cluster_id,instance)
            / (1024*1024) > 50
        for: 10m
        labels:
          tier: dedicated
          severity: major
          response_class: ticket
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3865
        annotations:
          description: 'The write throughput of a TiFlash node has exceeded 50 MB/s and lasted for 10 minutes. source_tcoc: TCOC-3865 dash_board: TiFlash-Summary -> Storage Write Stall -> Write Throughput By Instance -> "write-xxx". One TiFlash node''s write throughput exceeds 50MiB/s and lasts for 10 minutes in cluster {{ $labels.cluster_id }}. This may indicate a sudden change in the business or a sudden increase in write traffic caused by DDL reorg. Please confirm whether the TiFlash raft-log synchronization delay has increased and take mitigation measures.'
          value: "{{ $value }}"
          summary: The write throughput of a TiFlash node has exceeded 50 MB/s and lasted for 10 minutes.
      - alert: TikvServiceIsBusyForBig3
        expr: |
          max by (cluster_id)(sum(rate(tikv_scheduler_too_busy_total{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}[2m])) by (instance,cluster_id))>0
        for: 2m
        labels:
          tier: dedicated
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3748
        annotations:
          summary: Tikv service is busy
          description: 'Tikv service is busy. source_tcoc: TCOC-3748 dash_board: TiKV-Details -> Errors -> Service Is Busy. cluster_id: {{ $labels.cluster_id }}'
          value: "{{ $value }}"
      - alert: TiKVRaftstoreThreadCPUSecondsTotalForBig3
        expr: |
          sum(rate(tikv_thread_cpu_seconds_total{
            tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
            project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
            name=~"(raftstore|rs)_.*"
          }[1m])) by (cluster_id,instance,name)
            > 0.9
        for: 20m
        labels:
          tier: dedicated
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3877
        annotations:
          description: 'TiKV raftstore threads CPU usage > 90% for more than 10 minute. source_tcoc: TCOC-3877 dash_board: TiDB-Cluster-Clinic -> Storage Write Stall -> Query Summary -> Duration. cluster_id: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}'
          value: "{{ $value }}"
          summary: TiKV raftstore threads CPU usage > 90% for more than 10 minute.
      - alert: TiKVGrpcThreadHighCPUForBig3
        expr: |
          sum(rate(tikv_thread_cpu_seconds_total{
            tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
            project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
            name=~"grpc.*"
          }[1m])) by (cluster_id,instance, name)
          > 0.9
        for: 20m
        labels:
          tier: dedicated
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3877
        annotations:
          description: 'TiKV gRPC thread CPU usage > 90% for 10 minute source_tcoc: TCOC-3877 dashboard: TiDB-Cluster-Clinic → Storage Write Stall → Query Summary → Duration. cluster_id: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}, name: {{ $labels.name }}'
          value: "{{ $value }}"
          summary: TiKV gRPC thread CPU usage > 90% for more than 10 minute.
      - alert: TiFlashHighRaftWaitIndexForBig3
        expr: |
          histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054", project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}[2m])) by (le,cluster_id,instance))>60
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3968
        annotations:
          summary: The Raft index of a single TiFlash exceeds 60s and persists for more than 5 minute.
          message: >
            The Raft index of a single TiFlash exceeds 60s and persists for more than 5 minute.
            TiDB Cluster: {{ $labels.cluster_id }}
            source_tcoc: TCOC-3968
            dashboard: TiFlash-Summary -> Raft -> Raft Wait Index Duration
      - alert: TiFlashProxyEntryCacheHighMemoryForBig3
        expr: |
          ( tiflash_proxy_tikv_server_mem_trace_sum{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
            project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", name="raftstore-entry_cache"}
              / on(instance) tiflash_proxy_process_resident_memory_bytes{tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
            project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660"}
          ) > 0.2
        for: 0s
        labels:
          severity: major
          response_class: ticket
          component: tiflash
          oncall_team: tiflash
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-3940
        annotations:
          summary: The entry cache usage of TiFlash Proxy exceeds 20% of the memory.
          value: "{{ $value }}"
          message: >
            The entry cache usage of TiFlash Proxy exceeds 20% of the memory.
            source_tcoc: TCOC-3940
            dashboard: TiFlash-Proxy-Detail -> Cluster -> Raft Entry Cache
      - alert: KubePersistentVolumeFillingUpForBig3
        expr: |
          kubelet_volume_stats_available_bytes{
          tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
          project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
          job="kubelet",persistentvolumeclaim!~".*victoria-metrics-agent.*|.*vmagent.*|.*ng-monitoring.*|.*index-acceleration|vmstorage-volume-clinic-victoria-metrics-cluster-vmstorage.*",provider_type!~"aws-free-tier|alicloud-serverless"}
          /
          kubelet_volume_stats_capacity_bytes{
          tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
          project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
          job="kubelet",persistentvolumeclaim!~".*victoria-metrics-agent.*|.*vmagent.*|.*ng-monitoring.*|.*index-acceleration|vmstorage-volume-clinic-victoria-metrics-cluster-vmstorage.*",provider_type!~"aws-free-tier|alicloud-serverless"}
          < 0.15
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: resilience
          oncall_team: resilience
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4016
        annotations:
          summary: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage}} free.
          message: >
            The PersistentVolume claimed by {{ $labels.persistentvolumeclaim}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage}} free.
            source_tcoc: TCOC-4016
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
      - alert: TiKVCopGrpcP99DurationOver5sFor3minForBig3
        expr: |
          max by(cluster_id)(
          histogram_quantile(0.99, 
          sum(rate(tikv_grpc_msg_duration_seconds_bucket{
          tenant=~".*1372813089209061633|.*1372813089209289635",
          project!~"1372813089454536384",
          type!="kv_gc", type = "coprocessor"}[2m])) by (le, instance ,cluster_id)))>5
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: tikv
          oncall_team: tikv_unified
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4100
        annotations:
          summary: TiKV 99 cop grpc duration exceeds 5s and last for 3min. Hotspot or slow log may happen in {{ $labels.cluster_id }}.
          value: "{{ $value }}"
          message: >
            TiKV 99 cop grpc duration exceeds 5s and last for 3min. Hotspot or slow log may happen in {{ $labels.cluster_id }}.
            source_tcoc: TCOC-4100
            dashboard: TiKV-Details-Cluster-Clinic -> gRPC -> 99% gRPC message duration
#           - alert: TiKVCopP99DurationOutlierFor15mForBig3
#             expr: |
#               max by (cluster_id) (
#                 histogram_quantile(
#                   0.99,
#                   sum(
#                     rate(
#                       tikv_grpc_msg_duration_seconds_bucket{
#                         tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
#                         project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
#                         type="coprocessor"
#                       }[2m]
#                     )
#                   ) by (le, instance, cluster_id)
#                 )
#               )
#               >
#               7 *
#               avg(
#                 histogram_quantile(
#                   0.99,
#                   sum(
#                     rate(
#                       tikv_grpc_msg_duration_seconds_bucket{
#                         tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
#                         project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
#                         type="coprocessor"
#                       }[2m]
#                     )
#                   ) by (le, instance, cluster_id)
#                 )
#               )
#             for: 15m
#             labels:
#               severity: major
#               component: tikv
#               stability_governance: kernel-tcocp
#               source_tcoc: TCOC-4203
#             annotations:
#               summary: "TiKV coprocessor gRPC duration exceeds 7× cluster average for 15 minutes ({{ $labels.cluster_id }})"
#               value: '{{ $value }}'
#               message: >
#                 TiKV coprocessor gRPC duration exceeds 7× cluster average for 15 minutes ({{ $labels.cluster_id }})
#                 source_tcoc: TCOC-4203
#                 dashboard: TiKV-Details-Cluster-Clinic -> gRPC -> 99% gRPC message duration
      - alert: TiDBClusterBackupDurationTooLongCriticalForBig3
        expr: |
          dataflow_backup_execution_status_duration_seconds{
          tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
          project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
          status!~"failed|success|skipped|terminated", 
          } > 60 * 60 * 20
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: br
          oncall_team: tidb_x_tools
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4292
        annotations:
          summary: The TiDB Cluster {{ $labels.cluster_id }} backup running duration more than 20h.
          value: "{{ $value }}"
          message: >
            The TiDB Cluster {{ $labels.cluster_id }} backup running duration more than 20h.
            source_tcoc: TCOC-4292
      - alert: TiDBClusterRestoreDurationCriticalForBig3
        expr: |
          dataflow_restore_execution_status_duration_seconds{
          tenant=~".*1372813089209238420|.*1372813089209061633|.*1372813089209289635|.*1372813089209261054",
          project!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660",
          status!~"failed|success|finished|deleted", 
          } > 60 * 60 * 20
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: br
          oncall_team: tidb_x_tools
          stability_governance: kernel-tcocp
          source_tcoc: TCOC-4292
        annotations:
          summary: The TiDB Cluster {{ $labels.cluster_id }} backup restore duration more than 20h.
          value: "{{ $value }}"
          message: >
            The TiDB Cluster {{ $labels.cluster_id }} backup restore duration more than 20h.
            source_tcoc: TCOC-4292