groups:
- name: next-gen-k8s-kernel-tcocp.rules
  rules:
  - alert: KubePersistentVolumeFillingUpNextGen
    expr: |-
      (
      kubelet_volume_stats_available_bytes{job="kubelet",persistentvolumeclaim!~".*victoria-metrics-agent.*|.*vmagent.*|.*ng-monitoring.*|.*index-acceleration|vmstorage-volume-clinic-victoria-metrics-cluster-vmstorage.*|data-db-.*-tikv-.*|.*write-tiflash.*",provider_type!~"aws-free-tier|alicloud-serverless",persistentvolumeclaim!~".*compute-tiflash.*"}
        /
          kubelet_volume_stats_capacity_bytes{job="kubelet",persistentvolumeclaim!~".*victoria-metrics-agent.*|.*vmagent.*|.*ng-monitoring.*|.*index-acceleration|vmstorage-volume-clinic-victoria-metrics-cluster-vmstorage.*|data-db-.*-tikv-.*|.*write-tiflash.*",provider_type!~"aws-free-tier|alicloud-serverless",persistentvolumeclaim!~".*compute-tiflash.*"}
      <
        0.15
      )
      and
      (
      kubelet_volume_stats_available_bytes{job="kubelet",persistentvolumeclaim!~".*victoria-metrics-agent.*|.*vmagent.*|.*ng-monitoring.*|.*index-acceleration|vmstorage-volume-clinic-victoria-metrics-cluster-vmstorage.*|data-db-.*-tikv-.*|.*write-tiflash.*",provider_type!~"aws-free-tier|alicloud-serverless",persistentvolumeclaim!~".*compute-tiflash.*"}
      < 800 * 1024 * 1024 * 1024
      )
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: resilience
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: k8s
      oncall_team: resilience
      source_tcoc: TCOC-4016
    annotations:
      summary: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage}} free.
      message: |
        The PersistentVolume claimed by {{ $labels.persistentvolumeclaim}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage}} free. source_tcoc: TCOC-4016
  - alert: KubePersistentVolumeFillingUpTiFlashComputeNextGen
    expr: |-
      kubelet_volume_stats_available_bytes{job="kubelet",provider_type!~"aws-free-tier|alicloud-serverless",persistentvolumeclaim=~".*compute-tiflash.*"}
        /
          kubelet_volume_stats_capacity_bytes{job="kubelet",provider_type!~"aws-free-tier|alicloud-serverless",persistentvolumeclaim=~".*compute-tiflash.*"}
      <
        0.1
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: resilience
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: k8s
      oncall_team: resilience
      source_tcoc: TCOC-4016
    annotations:
      summary: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage}} free.
      message: |
        The PersistentVolume claimed by {{ $labels.persistentvolumeclaim}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage}} free. source_tcoc: TCOC-4016
  - alert: LightningPodNotReadyNextGen
    expr: sum(kube_pod_status_phase{job="kube-state-metrics",phase=~"Pending|Unknown",pod=~"import.*"}) > 0
    for: 15m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: lightning
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: k8s
      oncall_team: tidb_x_tools
      source_tcoc: TCOC-3817
    annotations:
      summary: Lightning Pod is not ready for 15 min
      message: Lightning of Tenant {{ $labels.tenant}}, Cluster {{ $labels.namespace }} has been in a non-ready state for longer than 15 minutes.
  - alert: UnhealthyTiDBComponentPodNextGen
    expr: kube_pod_status_phase{phase=~"Pending|Unknown|Failed",pod=~"db-[a-z0-9]+-(.+-)?(pd|tidb|tikv|tiflash|tiproxy)-.+"} == 1
    for: 20m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: kubernetes
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: k8s
      oncall_team: premium
      type: shoot
      source_tcoc: TCOC-3492
    annotations:
      message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 20 minutes.
  - alert: TiDBOOMRelatedPodRestartsNextGen
    expr: |-
      sum by (namespace, pod) (
        increase(kube_pod_container_status_restarts_total{container="tidb",pod=~"db-[0-9]+-tidb-.+"}[15m]) > 0
      )
      * on (namespace, pod) group_left ()
        max by (namespace, pod) (
          kube_pod_container_status_last_terminated_reason{container="tidb",pod=~"db-[0-9]+-tidb-.+",reason="OOMKilled"} > 0
        )
    for: 0s
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: tidb-executor
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: k8s
      oncall_team: premium
      source_tcoc: TCOC-4157
    annotations:
      message: |
        TiDB Pod {{ $labels.namespace }}/{{ $labels.pod }} restarted due to OOMKilled. source_tcoc: TCOC-4157