groups:
- name: next-gen-tikv-kernel-governance.rules
  rules:
  - alert: TiKVCPUCoreUsageOver80For5mNextGen
    expr: |-
      100
        *
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~".*(tikv|copr).*", job=~".*(tikv|copr).*"}[1m])
              )
              /
              on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_server_cpu_cores_quota{component=~".*(tikv|copr).*", job=~".*(tikv|copr).*"}
          )
      >
        80
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV single node CPU usage over 80%
      description: 'TiKV single node CPU usage exceeds 80% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
  - alert: TiKVCPUCoreUsageOver80For20mNextGen
    expr: |-
      100
        *
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~".*(tikv|copr).*", job=~".*(tikv|copr).*"}[1m])
              )
              /
              on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_server_cpu_cores_quota{component=~".*(tikv|copr).*", job=~"..*(tikv|copr).*"}
          )
      >
        80
    for: 20m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV single node CPU usage over 90%
      description: 'TiKV single node CPU usage exceeds 90% of total core usage. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
  - alert: TiKVReadpoolCPUFullFor1mNextGen
    expr: |
      100 *
      avg by (instance) (
        rate(
          tikv_thread_cpu_seconds_total{
            name=~"unified_read_po.*"
          }[1m]
        )
      ) > 95
    for: 1m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV readpool CPU is full 
      description: 'TiKV single node readpool CPU usage exceeds 95% of total readpool CPU usage for more than 1 minutes. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
  - alert: TiKVReadpoolCPUFullFor5mNextGen
    expr: |
      100 *
      avg by (instance) (
        rate(
          tikv_thread_cpu_seconds_total{
            name=~"unified_read_po.*"
          }[1m]
        )
      ) > 95
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV readpool CPU is full 
      description: 'TiKV single node readpool CPU usage exceeds 95% of total readpool CPU usage for more than 5 minutes. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
  - alert: TiKVReadpoolCPUFullFor20mNextGen
    expr: |
      100 *
      avg by (instance) (
        rate(
          tikv_thread_cpu_seconds_total{
            name=~"unified_read_po.*"
          }[1m]
        )
      ) > 95
    for: 20m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV readpool CPU is full 
      description: 'TiKV single node readpool CPU usage exceeds 95% of total readpool CPU usage for more than 20 minutes. cluster: {{ $labels.cluster_id }}, value: {{ $value }}'
  - alert: TiKVCPUImbalanceHighFor5mNextGen
    expr: |-
      (
        (
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
            (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~"tikv", job=~".*tikv.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_server_cpu_cores_quota{component=~"tikv", job=~".*tikv.*"}
            )
          )
        )
        /
        (
          (
            avg by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~"tikv", job=~".*tikv.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_server_cpu_cores_quota{component=~"tikv", job=~".*tikv.*"}
            )
          )
        )
      ) > 1.5
      and
      (
        100 *
        max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
          (
            sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
              rate(process_cpu_seconds_total{component=~"tikv", job=~".*tikv.*"}[5m])
            )
            / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
            tikv_server_cpu_cores_quota{component=~"tikv", job=~".*tikv.*"}
          )
        )
      ) > 65
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV instance CPU usage is significantly higher than cluster average
      description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV nodes continuously for 5 minutes. One TiKV node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
  - alert: TiKVCoprWorkerImbalanceHighFor5mNextGen
    expr: |-
      (
        (
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
            (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~".*(copr).*", job=~".*(copr).*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component=~".*(copr).*", job=~".*(copr).*"}
            )
          )
        )
        /
        (
          (
            avg by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~".*(copr).*", job=~".*(copr).*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component=~".*(copr).*", job=~".*(copr).*"}
            )
          )
        )
      ) > 1.5
      and
      (
        100 *
        max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
          (
            sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
              rate(process_cpu_seconds_total{component=~".*(copr).*", job=~".*(copr).*"}[5m])
            )
            / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
            tikv_worker_cpu_cores_quota{component=~".*(copr).*", job=~".*(copr).*"}
          )
        )
      ) > 65
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV Copr Worker CPU usage is significantly higher than cluster average
      description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV Copr Worker nodes continuously for 5 minutes. One TiKV Copr Worker node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
  - alert: TiKVWorkerImbalanceHighFor5mNextGen
    expr: |-
      (
        (
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
            (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component="tikv-worker", job=~".*tikv-worker.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component="tikv-worker", job=~".*tikv-worker.*"}
            )
          )
        )
        /
        (
          (
            avg by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component="tikv-worker", job=~".*tikv-worker.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component="tikv-worker", job=~".*tikv-worker.*"}
            )
          )
        )
      ) > 1.5
      and
      (
        100 *
        max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
          (
            sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
              rate(process_cpu_seconds_total{component="tikv-worker", job=~".*tikv-worker.*"}[5m])
            )
            / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
            tikv_worker_cpu_cores_quota{component="tikv-worker", job=~".*tikv-worker.*"}
          )
        )
      ) > 65
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV Worker CPU usage is significantly higher than cluster average
      description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV Worker nodes continuously for 5 minutes. One TiKV Worker node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
  - alert: TiKVCPUImbalanceHighFor20mNextGen
    expr: |-
      (
        (
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
            (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~"tikv", job=~".*tikv.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_server_cpu_cores_quota{component=~"tikv", job=~".*tikv.*"}
            )
          )
        )
        /
        (
          (
            avg by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~"tikv", job=~".*tikv.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_server_cpu_cores_quota{component=~"tikv", job=~".*tikv.*"}
            )
          )
        )
      ) > 1.5
      and
      (
        100 *
        max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
          (
            sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
              rate(process_cpu_seconds_total{component=~"tikv", job=~".*tikv.*"}[5m])
            )
            / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
            tikv_server_cpu_cores_quota{component=~"tikv", job=~".*tikv.*"}
          )
        )
      ) > 65
    for: 20m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV instance CPU usage is significantly higher than cluster average
      description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV nodes continuously for 20 minutes. One TiKV node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
  - alert: TiKVCoprWorkerImbalanceHighFor20mNextGen
    expr: |-
      (
        (
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
            (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~".*(copr).*", job=~".*(copr).*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component=~".*(copr).*", job=~".*(copr).*"}
            )
          )
        )
        /
        (
          (
            avg by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component=~".*(copr).*", job=~".*(copr).*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component=~".*(copr).*", job=~".*(copr).*"}
            )
          )
        )
      ) > 1.5
      and
      (
        100 *
        max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
          (
            sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
              rate(process_cpu_seconds_total{component=~".*(copr).*", job=~".*(copr).*"}[5m])
            )
            / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
            tikv_worker_cpu_cores_quota{component=~".*(copr).*", job=~".*(copr).*"}
          )
        )
      ) > 65
    for: 20m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV Copr Worker CPU usage is significantly higher than cluster average
      description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV Copr Worker nodes continuously for 20 minutes. One TiKV Copr Worker node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
  - alert: TiKVWorkerImbalanceHighFor20mNextGen
    expr: |-
      (
        (
          max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
            (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component="tikv-worker", job=~".*tikv-worker.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component="tikv-worker", job=~".*tikv-worker.*"}
            )
          )
        )
        /
        (
          (
            avg by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
              sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
                rate(process_cpu_seconds_total{component="tikv-worker", job=~".*tikv-worker.*"}[5m])
              )
              / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
              tikv_worker_cpu_cores_quota{component="tikv-worker", job=~".*tikv-worker.*"}
            )
          )
        )
      ) > 1.5
      and
      (
        100 *
        max by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id) (
          (
            sum by (tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance) (
              rate(process_cpu_seconds_total{component="tikv-worker", job=~".*tikv-worker.*"}[5m])
            )
            / on(tenant_id, tidb_cluster_id, cluster_id, sharedpool_id, instance)
            tikv_worker_cpu_cores_quota{component="tikv-worker", job=~".*tikv-worker.*"}
          )
        )
      ) > 65
    for: 20m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tikv
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: tikv_unified
    annotations:
      summary: TiKV Worker CPU usage is significantly higher than cluster average
      description: 'Cluster {{ $labels.cluster_id }} has experienced CPU load imbalance on TiKV Worker nodes continuously for 20 minutes. One TiKV Worker node''s CPU usage exceeds 65% and is 1.5x higher than the cluster''s average CPU usage. Value: {{ $value }}'
  - alert: TiKVDowntimeOver30MinNextGen
    expr:  up{component=~"tikv", job=~".*tikv.*", job!=".*worker.*", } == 0
    for: 30m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: infra-provider
      stability_governance: next-gen-kernel-governance
      visibility: internal
      source_component: tikv
      oncall_team: premium
    annotations:
      summary: '{{ $labels.component }} server has been down for over 30 minutes'
      description: 'Cluster: {{ $labels.cluster_id }}, Instance: {{ $labels.instance }} (Component: {{ $labels.component }}) is down for more than 30 minutes.'