groups:
  - name: nextgen-tikv-errorlog-governance
    interval: 30s
    rules:
    - alert: TiKVReadDefaultCFButDefaultValueNotFoundNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `mod.rs`
              |= `default value not found`
            [1m]
          ))
        ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "Default CF data is missing when reading data"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "default value not found" Affects the correctness of business data and has the risk of panic. Check whether the cluster has lost data due to network or disk failures. If the problem cannot be solved, contact R&D'
    - alert: TiKVTLSHandshakeErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `mod.rs`
              |= `Status server error: TLS handshake error`
            [1m]
          ))
        ) > 1
      for: 4m
      labels:
        severity: major
        response_class: ticket
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "TLS handshake error"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "Status server error: TLS handshake error" Failed attempts to automatically repair the problem will result in the inability to connect to the node, making the node unavailable. It is recommended to check whether the network connection is smooth, whether the firewall blocks the communication, and the certificate authority. If the problem cannot be solved, contact the R&D personnel'
    - alert: TiKVExceedsRaftEntryMaxSizeNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `peer.rs`
              |= `entry is too large`
            [1m]
          ))
        ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "The write log size exceeds raft-entry-max-size"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "entry is too large" After confirmation, consider whether to adjust the raft-entry-max-size parameter'
    - alert: TiKVFailedFetchSafepointFromPDNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `gc_manager.rs`
              |= `failed to get safe point from pd`
            [1m]
          ))
        ) > 4
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "Failed to obtain checkpoint from PD"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to get safe point from pd" Continuous occurrence may cause GC failure, resulting in redundant node space accumulation. This is usually caused by a problem with the PD communication. It is recommended to check the PD connection and status. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D'
    - alert: TiVKFailedToWatchResourceGroupsNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `service.rs`
              |= `failed to send significant msg`
            [1m]
          ))
        ) > 1
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "Unable to obtain PD information"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to send significant msg" If this happens for a long time, the node may become unavailable. This is usually caused by a problem with the PD communication. It is recommended to check the PD connection and status. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D'
    - alert: TiKVIngestFailNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `apply.rs`
              |= `ingest fail`
            [1m]
          ))
        ) > 4
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "TiKV Ingest Fail"
        message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "ingest fail" errors more than 4 times in the last 1 minute. This indicates **data ingestion failure**, typically due to **region version mismatches**. Occasional occurrences may self-recover. Frequent occurrences could **impact data import**. If this issue arises, first **check the node''s status**, including load and network communication. If it does not self-recover, **examine the specific error type in the logs and contact R&D**.'
    - alert: TiKVResolveStoreAddressFailedNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `raft_client.rs`
              |= `resolve store address failed`
              !~ "StoreTombstone"
            [1m]
          ))
        ) > 4
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "TiKV Resolve Store Address Failed"
        message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "resolve store address failed" errors more than 4 times in the last 1 minute. The inability to resolve store addresses, if persistent, will **lead to KV node failures**. This could be due to **disk offline events** or **old metadata not being properly cleaned up** after a disk was decommissioned. First, check if the disk is offline. If the disk is mounted and the error log shows a `store_id` mismatch, try using `tikv_ctl remove-fail-stores` or `pd_ctl store delete` to remove the old `store_id` information. If unresolved, **contact R&D**.'
    - alert: BRBackupCreateStorageFailedNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tikv"}
              |= `endpoint.rs`
              |= `backup create storage failed`
            [1m]
          ))
        ) > 0
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tikv
        oncall_team: tikv_unified
        stability_governance: nextgen-errorlog-governance
        tier: next-gen
      annotations:
        summary: "BRBackup Create Storage Failed"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "backup create storage failed"'