rule_files:
  - ../rules/cluster-next-gen/k8s/kernel-tcocp.rules.yaml
  - ../rules/cluster-next-gen/tikv/data-plane/kernel-tcocp.rules.yaml
  - ../rules/cluster-next-gen/tikv/data-plane/tikv.rules.yaml
  - ../rules/cluster-next-gen/tiflash/data-plane/tiflash.rules.yaml

evaluation_interval: 1m

tests:
  - name: Unhealthy NextGen component pods fire alerts
    interval: 1m
    input_series:
      - series: 'kube_pod_status_phase{namespace="tidb123",pod="db-a924bb0f-tidb-abcdef",cluster="shoot",phase="Pending"}'
        values: '1x25'
      - series: 'kube_pod_status_phase{namespace="tidb123",pod="db-a924bb0f-write-tiflash-abcdef",cluster="shoot",phase="Failed"}'
        values: '1x25'
    alert_rule_test:
      - eval_time: 21m
        alertname: UnhealthyTiDBComponentPodNextGen
        exp_alerts:
          - exp_labels:
              namespace: tidb123
              pod: db-a924bb0f-tidb-abcdef
              cluster: shoot
              phase: Pending
              tier: next-gen
              severity: critical
              response_class: page
              component: kubernetes
              stability_governance: next-gen-kernel-tcocp
              visibility: internal
              source_component: k8s
              oncall_team: premium
              type: shoot
              source_tcoc: TCOC-3492
            exp_annotations:
              message: Pod tidb123/db-a924bb0f-tidb-abcdef in shoot has been in a non-ready state for longer than 20 minutes.
          - exp_labels:
              namespace: tidb123
              pod: db-a924bb0f-write-tiflash-abcdef
              cluster: shoot
              phase: Failed
              tier: next-gen
              severity: critical
              response_class: page
              component: kubernetes
              stability_governance: next-gen-kernel-tcocp
              visibility: internal
              source_component: k8s
              oncall_team: premium
              type: shoot
              source_tcoc: TCOC-3492
            exp_annotations:
              message: Pod tidb123/db-a924bb0f-write-tiflash-abcdef in shoot has been in a non-ready state for longer than 20 minutes.

  - name: TiKV EBS IOPS alert matches a NextGen TiKV pod
    interval: 1m
    input_series:
      - series: 'aws_ebs_csi_exceeded_iops_seconds_total{instance="node-1"}'
        values: '0 60 120 180 240 300'
      - series: 'kube_pod_info{namespace="tidb123",pod="db-a924bb0f-tikv-abcdef",node="node-1"}'
        values: '1x6'
    alert_rule_test:
      - eval_time: 4m
        alertname: EBSTiKVIOPSExceededNextGen
        exp_alerts:
          - exp_labels:
              cluster_id: 123
              tier: next-gen
              severity: major
              response_class: ticket
              component: tikv
              stability_governance: next-gen-kernel-tcocp
              visibility: internal
              source_component: tikv
              oncall_team: tikv_unified
              source_tcoc: TCOC-3905
            exp_annotations:
              summary: TiKV EBS IOPS throttled on cluster 123.
              message: |
                TiKV EBS IOPS throttled (100% of time) on cluster 123. Please evaluate whether EBS IOPS scaling is needed and contact the customer via TAM if necessary. source_tcoc: TCOC-3905

  - name: TiKV EBS throughput alert matches a NextGen TiKV pod
    interval: 1m
    input_series:
      - series: 'aws_ebs_csi_exceeded_tp_seconds_total{instance="node-1"}'
        values: '0 12 24 36 48 60 72 84'
      - series: 'kube_pod_info{namespace="tidb123",pod="db-a924bb0f-tikv-abcdef",node="node-1"}'
        values: '1x8'
    alert_rule_test:
      - eval_time: 7m
        alertname: EBSTiKVThroughputExceededNextGen
        exp_alerts:
          - exp_labels:
              cluster_id: 123
              tier: next-gen
              severity: major
              response_class: ticket
              component: tikv
              stability_governance: next-gen-kernel-tcocp
              visibility: internal
              source_component: tikv
              oncall_team: tikv_unified
              source_tcoc: TCOC-3905
            exp_annotations:
              summary: TiKV EBS throughput throttled on cluster 123.
              message: |
                TiKV EBS throughput throttled (20% of time) on cluster 123. Please evaluate whether EBS throughput scaling is needed and contact the customer via TAM if necessary. source_tcoc: TCOC-3905

  - name: TiKV inode alert matches a NextGen TiKV PVC
    interval: 1m
    input_series:
      - series: 'kubelet_volume_stats_inodes_used{namespace="tidb123",pod="db-a924bb0f-tikv-abcdef",persistentvolumeclaim="data-db-a924bb0f-tikv-abcdef"}'
        values: '91x7'
      - series: 'kubelet_volume_stats_inodes{namespace="tidb123",pod="db-a924bb0f-tikv-abcdef",persistentvolumeclaim="data-db-a924bb0f-tikv-abcdef"}'
        values: '100x7'
    alert_rule_test:
      - eval_time: 6m
        alertname: TiKVInodeUsageMoreThan90%
        exp_alerts:
          - exp_labels:
              namespace: tidb123
              pod: db-a924bb0f-tikv-abcdef
              persistentvolumeclaim: data-db-a924bb0f-tikv-abcdef
              severity: critical
              response_class: page
              tier: next-gen
              component: tikv
              oncall_team: tikv_unified
            exp_annotations:
              description: TiKV inode usage for PVC data-db-a924bb0f-tikv-abcdef in namespace tidb123 has exceeded 90% (0.91). High inode usage may prevent new file creation and affect TiKV operations. Please clean up unnecessary files or expand storage capacity to resolve the inode shortage.
              value: '0.91'
              runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit?tab=t.0#heading=h.w4zl0kl6ny53
              summary: TiKV inode usage more than 90%

  - name: TiFlash inode alert matches NextGen write and compute PVCs
    interval: 1m
    input_series:
      - series: 'kubelet_volume_stats_inodes_used{namespace="tidb123",pod="db-a924bb0f-write-tiflash-abcdef",persistentvolumeclaim="data-db-a924bb0f-write-tiflash-abcdef"}'
        values: '92x7'
      - series: 'kubelet_volume_stats_inodes{namespace="tidb123",pod="db-a924bb0f-write-tiflash-abcdef",persistentvolumeclaim="data-db-a924bb0f-write-tiflash-abcdef"}'
        values: '100x7'
      - series: 'kubelet_volume_stats_inodes_used{namespace="tidb123",pod="db-a924bb0f-compute-tiflash-ghijkl",persistentvolumeclaim="data-db-a924bb0f-compute-tiflash-ghijkl"}'
        values: '93x7'
      - series: 'kubelet_volume_stats_inodes{namespace="tidb123",pod="db-a924bb0f-compute-tiflash-ghijkl",persistentvolumeclaim="data-db-a924bb0f-compute-tiflash-ghijkl"}'
        values: '100x7'
    alert_rule_test:
      - eval_time: 6m
        alertname: TiFlashInodeUsageMoreThan90%
        exp_alerts:
          - exp_labels:
              namespace: tidb123
              pod: db-a924bb0f-write-tiflash-abcdef
              persistentvolumeclaim: data-db-a924bb0f-write-tiflash-abcdef
              severity: critical
              response_class: page
              tier: next-gen
              component: tiflash
              oncall_team: tiflash
            exp_annotations:
              description: TiFlash inode usage for PVC data-db-a924bb0f-write-tiflash-abcdef in namespace tidb123 has exceeded 90% (0.92). High inode usage may prevent new file creation and affect TiFlash operations. Please clean up unnecessary files or expand storage capacity to resolve the inode shortage.
              value: '0.92'
              runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
              summary: TiFlash inode usage more than 90%
          - exp_labels:
              namespace: tidb123
              pod: db-a924bb0f-compute-tiflash-ghijkl
              persistentvolumeclaim: data-db-a924bb0f-compute-tiflash-ghijkl
              severity: critical
              response_class: page
              tier: next-gen
              component: tiflash
              oncall_team: tiflash
            exp_annotations:
              description: TiFlash inode usage for PVC data-db-a924bb0f-compute-tiflash-ghijkl in namespace tidb123 has exceeded 90% (0.93). High inode usage may prevent new file creation and affect TiFlash operations. Please clean up unnecessary files or expand storage capacity to resolve the inode shortage.
              value: '0.93'
              runbook_url: https://pingcap.feishu.cn/wiki/ZqMDwonIYi1o91kbk7AcOBvTnig
              summary: TiFlash inode usage more than 90%