groups:
- name: next-gen-tikv-kernel-tcocp.rules
  rules:
  - alert: TiKVMemoryCoreUsageOver85NextGen
    expr: |-
      max(
        100
        *
        (
          process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
          /
          on(instance) group_left()
          tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
        )
      ) > 85
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-2913
    annotations:
      message: |
        TiKV node memory utilization over 85%. value: {{ $value }} source_tcoc: TCOC-2913
  - alert: TiKVMemoryCoreUsageOver90NextGen
    expr: |-
      100
        *
        (
          process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
          /
          on(instance) group_left()
          tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
        )
      > 90
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-2913
    annotations:
      message: |
        TiKV node memory utilization over 90%. value: {{ $value }} source_tcoc: TCOC-2913
  - alert: TotalTiKVMemoryUsageOver80NextGen
    expr: |-
      avg(
        process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
        /
        on(instance) group_left()
        tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
      ) > 0.8
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-2913
    annotations:
      description: 'Total TiKV memory utilization over 80%. value: {{ $value }}'
      summary: 'Total TiKV memory utilization exceeds 80% for 10 minutes'
  - alert: TiKVCoreMemoryUsageOverAvgTotalNextGen
    expr: |-
      max by (cluster_id, tenant) (
        (
          100
          *
          (
            process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
            /
            on(instance) group_left()
            tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
          )
        )
        and
        (
          (
            100
            *
            (
              process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
              /
              on(instance) group_left()
              tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
            )
          )
          >
            (
              100
              *
              avg by (cluster_id, tenant) (
                process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
                /
                on(instance) group_left()
                tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
              )
              +
              10
            )
        )
        and
        (
          100
          *
          (
            process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
            /
            on(instance) group_left()
            tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
          )
        ) > 70
      )
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-2913
    annotations:
      description: 'TiKV single node memory utilization over (Total TiKV node memory utilization + 10). value: {{ $value }} source_tcoc: TCOC-2913'
      summary: 'TiKV single node memory utilization exceeds the cluster average by 10 points'
  - alert: TiDBClientLatencyExceeds1s
    expr: |-
      max(
          histogram_quantile(
            0.99,
            sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{instance=~".*",scope="false"}[1m]))
          )
        )
      >
        1
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: GTOC-7734
    annotations:
      summary: TiDB client p99 latency > 1s (instant alert)
      description: 'TiDB client p99 latency > 1s (instant alert) source_tcoc: GTOC-7734 grafana_dashboard: TiDB-Cluster-Clinic / KV Request / RPC Layer Latency P99. store: {{ $labels.store }}, value: {{ $value }}'
      value: '{{ $value }}'
  - alert: TiDBClientLatencyExceeds200ms
    expr: |-
      max(
          histogram_quantile(
            0.99,
            sum(rate(tidb_tikvclient_rpc_net_latency_seconds_bucket{instance=~".*",scope="false"}[1m]))
          )
        )
      >
        0.2
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: GTOC-7734
    annotations:
      summary: TiDB client p99 latency > 200ms for 5 minute
      description: 'TiDB client p99 latency > 200ms for 5 minute source_tcoc: GTOC-7734 grafana_dashboard: TiDB-Cluster-Clinic / KV Request / RPC Layer Latency P99. store: {{ $labels.store }}, value: {{ $value }}'
      value: '{{ $value }}'
  - alert: TikvServiceIsBusyNextGen
    expr: |-
      label_replace(
                  sum(rate(tikv_scheduler_too_busy_total{instance=~"$instance"}[2m])),
                  "metric",
                  "scheduler_too_busy",
                  "",
                  ""
                )
              or
                label_replace(
                  sum(rate(tikv_channel_full_total{instance=~"$instance"}[2m])),
                  "metric",
                  "channel_full",
                  "",
                  ""
                )
            or
              label_replace(
                sum(rate(tikv_coprocessor_request_error{instance=~"$instance",type="full"}[2m])),
                "metric",
                "coprocessor_error_full",
                "",
                ""
              )
          or
            label_replace(
              avg(tikv_engine_write_stall{db=~"$db",instance=~"$instance",type="write_stall_percentile99"}),
              "metric",
              "write_stall",
              "",
              ""
            )
        or
          label_replace(
            sum(
              rate(tikv_raftstore_store_write_msg_block_wait_duration_seconds_count{instance=~"$instance"}[2m])
            ),
            "metric",
            "raftstore_write_block_wait",
            "",
            ""
          )
      or
        label_replace(
          sum(tikv_raftstore_process_busy{instance=~"$instance"}),
          "metric",
          "raftstore_process_busy",
          "",
          ""
        )
    for: 0s
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-3748
    annotations:
      summary: Tikv service is busy
      description: 'Tikv service is busy. source_tcoc: TCOC-3748 dash_board: TiKV-Details -> Errors -> Service Is Busy. cluster_id: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}'
      value: '{{ $value }}'
  - alert: TiKVRaftstoreThreadCPUUsageOver90NextGen
    expr: avg by (instance) (rate(tikv_thread_cpu_seconds_total{name=~"(raftstore|rs|rfstore)_.*"}[1m])) > 0.9
    for: 20m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-3877
    annotations:
      summary: TiKV raftstore threads CPU usage > 90% for more than 10 minute.
      description: 'TiKV raftstore threads average CPU usage > 90% for more than 10 minute. source_tcoc: TCOC-3877 dash_board: TiDB-Cluster-Clinic -> Storage Write Stall -> Query Summary -> Duration. cluster_id: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}'
      value: '{{ $value }}'
  - alert: TiKVGrpcThreadHighCPUNextGen
    expr: avg by (instance) (rate(tikv_thread_cpu_seconds_total{name=~"grpc.*"}[1m])) > 0.9
    for: 20m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-3877
    annotations:
      summary: TiKV gRPC threads CPU usage > 90% for more than 10 minute.
      description: 'TiKV gRPC threads average CPU usage > 90% for more than 10 minute. source_tcoc: TCOC-3877 dashboard: TiDB-Cluster-Clinic → Storage Write Stall → Query Summary → Duration. cluster_id: {{ $labels.cluster_id }}, instance: {{ $labels.instance }}'
      value: '{{ $value }}'
  - alert: TiKVCopGrpcP99DurationOver5sFor3minNextGen
    expr: |-
      max(
          histogram_quantile(
            0.99,
            sum(rate(tikv_grpc_msg_duration_seconds_bucket{type!="kv_gc",type="coprocessor"}[2m]))
          )
        )
      >
        5
    for: 3m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-4100
    annotations:
      summary: TiKV 99 cop grpc duration exceeds 5s and last for 3min. Hotspot or slow log may happen in {{ $labels.cluster_id }}.
      message: |
        TiKV 99 cop grpc duration exceeds 5s and last for 3min. Hotspot or slow log may happen in {{ $labels.cluster_id }}. source_tcoc: TCOC-4100 dashboard: TiKV-Details-Cluster-Clinic -> gRPC -> 99% gRPC message duration
      value: '{{ $value }}'
  - alert: EBSTiKVIOPSExceededNextGen
    expr: |-
      max by(cluster_id)(
        label_replace(
          max by(namespace)(
            label_replace(
              rate(aws_ebs_csi_exceeded_iops_seconds_total[2m]),
              "node", "$1", "instance", "(.+)"
            )
            * on(node) group_right()
            kube_pod_info{pod=~"db-[a-z0-9]+-tikv-.+"}
          ) > 0.5,
          "cluster_id", "$1", "namespace", "tidb(.*)"
        )
      )
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-3905
    annotations:
      summary: TiKV EBS IOPS throttled on cluster {{ $labels.cluster_id }}.
      message: |
        TiKV EBS IOPS throttled ({{ $value | humanizePercentage }} of time) on cluster {{ $labels.cluster_id }}. Please evaluate whether EBS IOPS scaling is needed and contact the customer via TAM if necessary. source_tcoc: TCOC-3905
  - alert: EBSTiKVThroughputExceededNextGen
    expr: |-
      max by(cluster_id)(
        label_replace(
          max by(namespace)(
            label_replace(
              rate(aws_ebs_csi_exceeded_tp_seconds_total[2m]),
              "node", "$1", "instance", "(.+)"
            )
            * on(node) group_right()
            kube_pod_info{pod=~"db-[a-z0-9]+-tikv-.+"}
          ) > 0.1,
          "cluster_id", "$1", "namespace", "tidb(.*)"
        )
      )
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-3905
    annotations:
      summary: TiKV EBS throughput throttled on cluster {{ $labels.cluster_id }}.
      message: |
        TiKV EBS throughput throttled ({{ $value | humanizePercentage }} of time) on cluster {{ $labels.cluster_id }}. Please evaluate whether EBS throughput scaling is needed and contact the customer via TAM if necessary. source_tcoc: TCOC-3905
  - alert: TiKVOOMRelatedPodRestartsNextGen
    expr: |-
      sum by(namespace, pod) (
        increase(kube_pod_container_status_restarts_total{container="tikv",pod=~"db-[a-z0-9]+-tikv-.+"}[15m]) > 0
      )
      * on (namespace, pod) group_left ()
        max by(namespace, pod) (
          kube_pod_container_status_last_terminated_reason{container="tikv",pod=~"db-[a-z0-9]+-tikv-.+",reason="OOMKilled"} > 0
        )
    for: 0s
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-4157
    annotations:
      message: |
        TiKV Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled. source_tcoc: TCOC-4157
  - alert: EBSTiKVVolumeIOHangNextGen
    expr: |-
      label_replace(
        (
          (aws_ebs_csi_volume_queue_length > 0)
          * on(volume_id) group_left(persistentvolume, storageclass)
          label_replace(
            max by(csi_volume_handle, persistentvolume, storageclass) (
              kube_persistentvolume_info{csi_driver="ebs.csi.aws.com", storageclass=~".*tikv.*"}
            ),
            "volume_id", "$1", "csi_volume_handle", "(.+)"
          )
          * on(persistentvolume) group_left(persistentvolumeclaim, namespace)
          label_replace(
            max by(volumename, persistentvolumeclaim, namespace) (
              kube_persistentvolumeclaim_info
            ),
            "persistentvolume", "$1", "volumename", "(.+)"
          )
          * on(namespace, persistentvolumeclaim) group_left(pod)
          max by(namespace, persistentvolumeclaim, pod) (
            kube_pod_spec_volumes_persistentvolumeclaims_info
          )
        )
        unless on(instance, volume_id)
        (rate(aws_ebs_csi_write_ops_total[2m]) > 0)
        unless on(instance, volume_id)
        (rate(aws_ebs_csi_read_ops_total[2m]) > 0),
        "cluster_id", "$1", "namespace", "tidb(.*)"
      )
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-4157
    annotations:
      summary: EBS volume IO hang on TiKV pod (IO queued but not completing) - suspected EBS failure on cluster {{ $labels.cluster_id }}.
      message: |
        Suspected EBS volume failure: queue_length={{ $value }} but no read/write IO completing for 5min on pod {{ $labels.namespace }}/{{ $labels.pod }}, volume {{ $labels.volume_id }}, instance {{ $labels.instance }}. source_tcoc: TCOC-4157
  - alert: TotalTiKVMemoryUsageOver85NextGen
    expr: |-
      avg(
        process_resident_memory_bytes{component="tikv", job=~".*tikv.*"}
        /
        on(instance) group_left()
        tikv_server_memory_quota_bytes{component="tikv", job=~".*tikv.*"}
      ) > 0.85
    for: 10m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
      source_tcoc: TCOC-2913
    annotations:
      description: 'Total TiKV memory utilization over 85% for 10 minutes'
      summary: 'Total TiKV memory utilization exceeds 85% for 10 minutes'