groups:
  - name: nextgen-tidb-errorlog-governance
    interval: 30s
    rules:
        - alert: PDTsoRequestIsCanceledDueToTimeoutNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `tso_dispatcher.go`
                  |= `[tso] tso request is canceled due to timeout`
                [1m]
              ))
            ) > 4
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: pd
            oncall_team: pd
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Affect TSO service"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[tso] tso request is canceled due to timeout" If it is not a network error, you can contact oncall for troubleshooting'
        - alert: TiDBAutoAnalyzeFailedInitializeQueueNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `refresher.go`
                  |= `Failed to initialize the queue`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-optimizer
            oncall_team: tidb_optimizer
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Auto Analyze Queue Construction Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "Failed to initialize the queue" Unable to perform auto analyze for some time'
        - alert: TiDBRunDDLJobErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `[ddl] run DDL job error`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The DDL job execution failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[ddl] run DDL job error" Contact R&D to check the type and reason of the failed DDL'
        - alert: TiDBAdminCheckFoundDataInconsistencyNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `reporter.go`
                  |= `admin check found data inconsistency`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: critical
            response_class: page
            component: tidb-txn
            oncall_team: tidb_txn
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Data Accuracy Risk"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "admin check found data inconsistency" technical support required'
        - alert: TiDBAddIndexMayFailAndReportBuildIngestBackendFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `backend_mgr.go`
                  |= `build ingest backend failed`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Add index registration backend failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "build ingest backend failed" which may cause failure to add index. Contact R&D to check the reason'
        - alert: TiDBDXFFailedAndReportCheckWhetherThereAreSubtasksFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `task_executor.go`
                  |= `check whether there are subtasks to run failed`
                [3m]
              ))
            ) > 1
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The subtask executor failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "check whether there are subtasks to run failed" when checking whether there are subtasks that can be executed. This may cause add index or import into to fail. Contact R&D to check the reason'
        - alert: TiDBLoadDataFailedAndReportCommitErrorCheckAndInsertNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `load_data.go`
                  |= `commit error CheckAndInsert`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Load data failed to import in batches"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "commit error CheckAndInsert" The disk may be full or the transaction may fail. Contact R&D to check the specific problem'
        - alert: TiDBLoadDataFailedAndReportCommitErrorRefreshNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `load_data.go`
                  |= `commit error refresh`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The transaction submission of load data failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "commit error refresh" There may be duplicate keys and other problems. Contact R&D for inspection'
        - alert: TiDBRunSQLFailedAndReportCompileSQLPanicNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `compiler.go`
                  |= `compile SQL panic`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql
            oncall_team: tidb_sql
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Alarm requires specific investigation"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "compile SQL panic" find out why it panicked and then fixed'
        - alert: TiDBRunSQLMayFailAndReportConnectionRunningLoopPanicNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `conn.go`
                  |= `connection running loop panic`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql
            oncall_team: tidb_sql
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Alarm requires specific investigation"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "connection running loop panic" find out why it panicked and then fixed'
        - alert: TiDBDrainConnectionFailedInLoadDataNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `conn.go`
                  |= `drain connection failed in load data`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Load data When importing"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "drain connection failed in load data" an error occurs when reading a data packet from a connection. This may cause load data to fail. Contact R&D for inspection'
        - alert: TiDBDoImportFailedAndReportErrorOnEncodeAndSortNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `encode_and_sort_operator.go`
                  |= `error on encode and sort`
                  != `context canceled`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Import into reports an error"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "error on encode and sort" during the encode & sort phase. Contact R&D to check the cause'
        - alert: TiDBReportErrorWhenGettingDdlHistoryCountNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `ddl.go`
                  |= `error when getting the ddl history count`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Failure to obtain the DDL historical sequence number"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "error when getting the ddl history count" may cause problems such as abnormal owner status and abnormal DDL task scheduling. Contact R&D to check the cause'
        - alert: TiDBDXFFailedAndReportGetTaskFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `manager.go`
                  |= `get task failed`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The distributed framework failed to query the task from the system table"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "get task failed" The subsequent task scheduling logic cannot continue to execute, which may cause the failure of adding indexes or importing into. Contact R&D to check the cause'
        - alert: TiDBAddIndedMayFailAndReportImportErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `backfilling_operators.go`
                  |= `import error`
                  != `Duplicate entry`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The data import into TikV fails"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "import error" when adding indexes. This may be caused by duplicate keys, etc. It is recommended to contact R&D to check the cause'
        - alert: TiDBAddIndexFailedAndReportIngestDataIntoStorageErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (
              sum by (tidb_cluster_id, project_id, instance_id) (
                count_over_time(
                  {cluster_env="prod", project_id = "" , container="tidb"}
                    |= `backend.go`
                    |= `ingest data into storage error`
                  [1m]
                )
              ) > 0
              unless
              sum by (tidb_cluster_id, project_id, instance_id) (
                count_over_time(
                  {cluster_env="prod", project_id = "" , container="tidb"}
                    |= `found duplicate key`
                  [1m]
                )
              ) > 0
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Failure in importing indexed data into TikV"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "ingest data into storage error" may result in failure in indexing. It is recommended to contact R&D to check the cause. For specific reasons, see the error content in err id 76'
        - alert: TiDBLoadDataLocalFailNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `load_data.go`
                  |= `load local data failed`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "load data local Failed to import data"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "load local data failed" This may be due to problems such as the disk being full. Contact R&D to check the specific error content'
        - alert: TiDBInfoSchemReloadShemaFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `domain.go`
                  |= `reload schema in loop failed`
                [1m]
              ))
            ) > 4
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "info schema Failed to load the latest schema information"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "reload schema in loop failed" Continuous failures may cause queries to use old version schemas and report errors. It is recommended that you contact R&D to check the specific cause.'
        - alert: TiDBShemaSyncerRestartFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `domain.go`
                  |= `reload schema in loop, schema syncer restart failed`
                [1m]
              ))
            ) > 4
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB node synchronization schema version failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "reload schema in loop, schema syncer restart failed" the node may be unable to process DDL. It is recommended to contact R&D to check the etcd and network issues of the cluster'
        - alert: TiDBShemaSyncerRestartFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `domain.go`
                  |= `restart the schema syncer failed`
                [1m]
              ))
            ) > 2
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The TiDB node failed to synchronize the schema version"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "restart the schema syncer failed" the node cannot process DDL. It is recommended to contact R&D to check the etcd and network issues of the cluster'
        - alert: TiDBDXFFailedAndReportTaskRevertedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `handle.go`
                  |= `task reverted`
                  != `context canceled`
                  != `Duplicate entry`
                  != `Duplicate key`
                  != `Value conversion failed for column`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The distributed framework task execution failed and rolled back"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "task reverted" Adding indexes or importing into on the user side may fail. For specific reasons, it is recommended to contact R&D to check the specific error information. It may be that adding uk encountered duplicate keys and other problems'
        - alert: PDGetTSErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `tso_dispatcher.go`
                  |= `[tso] getTS error`
                [1m]
              ))
            ) > 9
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: pd
            oncall_team: pd
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TSO Request Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[tso] getTS error", which is usually related to leader switching or TCP connection disconnection. You can judge it through the log. If it is due to these two reasons, you can wait for self-recovery. If it is not an error, ask R&D support. If it is not recovered after more than 5 minutes, contact R&D'
        # Temporarily disabled due to too many false positives; may be re-enabled later.
        # - alert: TiDBAutoAnalyzeFailedNextGen
        #   expr: |
        #     max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
        #       count_over_time(
        #         {cluster_env="prod", project_id = "", container="tidb"}
        #           |~ "(update\\.go|exec\\.go|autoanalyze\\.go)"
        #           |= `auto analyze failed`
        #         [5m]
        #       ))
        #     ) > 0
        #     unless
        #     max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
        #       count_over_time(
        #         {cluster_env="prod", project_id = "", container="tidb"}
        #           |= `parallel cpu profiler stopped`
        #         [5m]
        #       ))
        #     ) > 0
        #   for: 0s
        #   labels:
        #     severity: major
        #     component: tidb-optimizer
        #     oncall_team: tidb_optimizer
        #     tier: next-gen
        #     stability_governance: nextgen-errorlog-governance
        #   annotations:
        #     summary: "TiDB auto analyze failed "
        #     message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "auto analyze failed". Please investigate by  the SOP: https://pingcap.feishu.cn/wiki/Fe4vwRa4iiCgYRkg6wec6PkQnlh'
        - alert: TiDBFailLoadSafepointFromPDNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "", container="tidb"}
                  |= `kv.go`
                  |= `fail to load safepoint from pd`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-txn
            oncall_team: tidb_txn
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Failed to get information from pd"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "fail to load safepoint from pd" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support'
        - alert: TiDBFailLoadSafepointFromPD4wetechsaas01NextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "", container="tidb"}
                  |= `kv.go`
                  |= `fail to load safepoint from pd`
                [1m]
              ))
            ) > 5
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-txn
            oncall_team: tidb_txn
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Failed to get information from pd"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "fail to load safepoint from pd" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support'
        - alert: TiDBDXFFailedAndReportFailedHandleTaskNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `task_executor.go`
                  |= `failed to handle task`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Distributed framework handles errors when running tasks"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to handle task" add index or import into may fail. Contact R&D to check'
        - alert: TiDBDXFFailedAndReportGetSubtasksFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `task_executor.go`
                  |= `get subtasks failed`
                  != `context canceled`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The distributed framework reports an error"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "get subtasks failed" When the distributed framework is balancing subtask scheduling, an error occurs when reading the tidb_background_subtask system table. Add index or import into may fail. Contact R&D for inspection'
        - alert: TiDBLoadStoreFromPDFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |~ "(store_cache\\.go|region_cache\\.go)"
                  |= `loadStore from PD failed`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-txn
            oncall_team: tidb_txn
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Failed to get information from pd"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "loadStore from PD failed" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support'
        - alert: TiDBUnexpectedPanicInRecoverableGoroutineNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `misc.go`
                  |= `panic in the recoverable goroutine`
                  !~ "memory limit"
                  !~ "Query execution was interrupted"
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql
            oncall_team: tidb_sql
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "A panic occurred online"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "panic in the recoverable goroutine" Need to know the specific panic stack, then open an issue to resolve it'
        - alert: TiDBImportIntoReportErrPostProcessFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `subtask_executor.go`
                  |= `post process failed`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "import into reports an error in the subtask during the post process phase"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "post process failed" The import into task may fail. Contact R&D for processing'
        - alert: TiDBDXFFailedAndReportRefreshTaskFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `task_executor.go`
                  |= `refresh task failed`
                  != `context canceled`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "An error occurred while reading the tidb_global_task system table when checking task status"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "refresh task failed" It can add index or import into may fail, contact R&D to check'
        - alert: TiDBImportIntoReportErrRunSubtaskFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `task_executor.go`
                  |= `run subtask failed`
                  != `context canceled`
                  != `Duplicate entry`
                  != `Duplicate key`
                  != `found duplicate key`
                  != `subtask not found`
                  != `Value conversion failed for column`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "import into reports an error in the post process phase"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "run subtask failed" The import into task may fail. Contact R&D for processing.'
        - alert: TiDBShuffleExecPanicNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `shuffle.go`
                  |= `shuffle panicked`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-executor
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "BUG"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "shuffle panicked" please contact R&D for the work order'
        - alert: TiDBDXFFailedAndReportTaskExecutorMetFirstErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `task_executor.go`
                  |= `taskExecutor met first error`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "The first error encountered when the distributed framework processes tasks"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "taskExecutor met first error" Import into or add index may fail. It is recommended to contact R&D to check the cause'
        - alert: TiDBLoadStoreFromPDFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `region_cache.go`
                  |= `loadStore from PD failed`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-txn
            oncall_team: tidb_txn
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB Load Store From PD Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "loadStore from PD failed" errors in the last 1 minute. This failure to get information from PD could be due to **network issues or PD restarts**. If PD cannot communicate, the cluster might **become unavailable** and other alerts should also be present. When this occurs, **investigate network connectivity and PD load**, and observe TiDB''s pd-client monitoring. Monitor for self-recovery. No parameter adjustments are needed. If it does not self-recover, escalate to technical support.'
        - alert: TiDBDXFFailedAndReportTaskManagerMetErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `manager.go`
                  |= `task manager met error`
                  != `error="context canceled"`
                [5m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB Task Manager Met Error"
            message: 'TiDB cluster {{ $labels.tidb_cluster_id }} reported "task manager met error". This indicates a failure in the **distributed framework processing tasks**, potentially causing **add index and import into tasks to fail**. Contact R&D to check the root cause of the task or subtask failure.'
        - alert: TiDBUpdateCheckpointFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `checkpoint.go`
                  |= `update checkpoint failed`
                [5m]
              ))
            ) > 20
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-sql-data
            oncall_team: ingress_service
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB Update Checkpoint Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "update checkpoint failed" errors. This occurs during **add index ingest**, preventing **ingest progress** and causing **add index failures**. Contact R&D to investigate the cause of the failure.'
        - alert: TiDBUpdateTSErrorNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `pd.go`
                  |= `updateTS error`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-txn
            oncall_team: tidb_txn
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB Update TSO Error"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "updateTS error" in the last 1 minute. This failure to get information from PD could be due to **network issues or PD restarts**. If PD cannot communicate, the cluster might **become unavailable** and other alerts should also be present. When this occurs, **investigate network connectivity and PD load**, and observe TiDB''s pd-client monitoring. Monitor for self-recovery. No parameter adjustments are needed. If it does not self-recover, escalate to technical support.'
        - alert: TiDBPlannerHandleDdlEventFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "",  container="tidb"}
                  |= `domain.go`
                  |= `handle ddl event failed`
                [5m]
              ))
            ) > 4
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-optimizer
            oncall_team: tidb_optimizer
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB Planner Handle DDL Event Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "handle ddl event failed" errors more than 4 times in the last 5 minutes. Currently, this error **is unlikely to affect business operations**. If the cluster load is high (e.g., increased query backoff), this error can be ignored. However, if the cluster load is low, this error is not expected by design and **requires R&D intervention**.'
        - alert: TiDBMetExpensiveQueryNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (
              sum by (tidb_cluster_id, project_id, instance_id) (
                count_over_time(
                  {cluster_env="prod", project_id = "",  container="tidb"}
                    |= `[expensive_query]`
                    != `sql="analyze`
                    != `sql="ANALYZE`                 
                  [10m]
                )
              )
            ) > 50
            and
            max by (tidb_cluster_id, project_id) (
              sum by (tidb_cluster_id, project_id, instance_id) (
                count_over_time(
                  {cluster_env="prod", project_id = "",  container="tidb"}
                    |= `tp add index`
                    |= `Type:modify column`
                  [10m]
                )
              )
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-optimizer
            oncall_team: tidb_optimizer
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Too much expensive queries"
            message: 'TiDB cluster {{ $labels.tidb_cluster_id }} just met too much expensive queries'
        - alert: TiDBExpensiveQueryCntGt100For1mNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (
              sum by (tidb_cluster_id, project_id, instance_id) (
                count_over_time(
                  {cluster_env="prod", project_id=~"1372813089454544954", project_id = "" , container="tidb"}
                    |= `[expensive_query]`
                    != `sql="analyze`
                    != `sql="ANALYZE`
                  [10m]
                )
              )
            ) > 100
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb-optimizer
            oncall_team: tidb_optimizer
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "A large number of running expensive queries exceeding 60 seconds."
            message: 'TiDB cluster {{ $labels.tidb_cluster_id }} just met 100+ expensive queries exceeding 1 minute.'
        - alert: TiDBExpensiveQueryCntGt500AndIncreaseFor10mNextGen
          expr: |
            (max by (tidb_cluster_id, project_id) (
                      sum by (tidb_cluster_id, project_id, instance_id) (
                        count_over_time(
                          {cluster_env="prod", project_id=~"1372813089454544954", project_id = "" , container="tidb"}
                            |= `[expensive_query]`
                            != `sql="analyze`
                          [10m]
                        )
                      ) 
                    ) > 500) and (max by (tidb_cluster_id, project_id) (
                count_over_time(
                  {cluster_env="prod", project_id=~"1372813089454544954", project_id = "" , container="tidb"}
                    |= `[expensive_query]`
                    != `sql="analyze`
                  [1m]
                )
              )
              >
              3 *
              max by (tidb_cluster_id, project_id) (
                count_over_time(
                  {cluster_env="prod", project_id=~"1372813089454544954", project_id = "" , container="tidb"}
                    |= `[expensive_query]`
                    != `sql="analyze`
                  [1m] offset 10m
                )
              ))
          for: 0s
          labels:
            severity: critical
            response_class: page
            component: tidb-optimizer
            oncall_team: tidb_optimizer
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "A large number of running expensive queries exceeding 10 min."
            message: 'TiDB cluster {{ $labels.tidb_cluster_id }} just met 500+ running expensive queries and is 3x higher than 10min before.'
        - alert: RebuildTableAnalysisJobQueueFailNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `autoanalyze.go`
                  |= `rebuild table analysis job queue failed`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Rebuild Table Analysis Job Queue Fail"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "rebuild table analysis job queue failed" The situation of excessive tikv pressure such as tikv unavailable can be ignored, and other clearing alarms will be sent directly. But this is not a serious problem'
        - alert: LightningImportFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `backend.go`
                  |= `import failed`
                  !~ `context canceled`
                  != `found duplicate key`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Lightning Import Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "import failed" Lightning import failed. Unless the user canceled it, please contact R&D to check the specific reason.'
        - alert: DDLCannotFindReorgCtxNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `ddl.go`
                  |= `cannot find reorgCtx`
                [1m]
              ))
            ) > 0
          for: 3m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "DDL Cannot Find Reorg Ctx"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "cannot find reorgCtx" reorgCtx is used to notify the reorg worker of the status change when the DDL add index or modify column rollback occurs. Continuous occurrence of this error may cause the add index or modify column rollback to fail.'
        - alert: FailedToUploadSafepointToPDNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `gc_worker.go`
                  |= `failed to upload safe point to PD`
                [1m]
              ))
            ) > 0
          for: 2m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Failed To Upload Safepoint To PD"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to upload safe point to PD"'
        - alert: LoadDataErrorAndReportCommitWorkPanickedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `load_data.go`
                  |= `commitWork panicked`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Load Data Error And Report Commit Work Panicked"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "commitWork panicked" When loading data, inserting data fails because the memory quota is exceeded. You can try temporarily increasing the mem limit or reducing the tidb_dml_batch_size and try again.'
        - alert: ImportIntoOrAddIndexReportScanRegionFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `local.go`
                  |= `scan region failed`
                  !~ `context canceled`
                [1m]
              ))
            ) > 0
          for: 1m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Import Into Or Add Index Report Scan Region Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "scan region failed"'
        - alert: RegionCacheLoadRegionFailedNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `region_cache.go`
                  |= `load region failure`
                [1m]
              ))
            ) > 0
          for: 2m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Region Cache Load Region Failed"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "load region failure"'
        - alert: ValidateReadTSFailedForRequestNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `region_request.go`
                  |= `validate read ts failed for request`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Validate Read TS Failed For Request"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "validate read ts failed for request"'
        - alert: RegionUnavailableNextGen
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "" , container="tidb"}
                  |= `txn.go`
                  |= `region unavailable`
                [1m]
              ))
            ) > 0
          for: 2m
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "Region Unavailable"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "region unavailable"'
        - alert: TiDBCoprKeyRangeSplitAnomaly
          expr: |
            max by (tidb_cluster_id, project_id) (sum by (tidb_cluster_id, project_id, instance_id) (
              count_over_time(
                {cluster_env="prod", project_id = "", container="tidb"}
                  |~ `(Request range exceeds bound|request range exceeds bound|LocateBucket returned nil|SplitKeyRangesByBuckets fell back to region-only splitting|skip-buckets SplitKeyRangesByLocations produced invalid location coverage|skip-buckets SplitKeyRangesByLocations produced out-of-bound ranges before sending|SplitKeyRangesByLocations: re-locate overflow budget exhausted|SplitKeyRangesByLocations: re-locating overflow ranges beyond pre-fetched locations|SplitKeyRangesByLocations: break early but remaining ranges start outside location|SplitKeyRangesByLocations: last location but ranges start outside)`
                [1m]
              ))
            ) > 0
          for: 0s
          labels:
            severity: warning
            component: tidb
            oncall_team: tidb_executor
            tier: next-gen
            stability_governance: nextgen-errorlog-governance
          annotations:
            summary: "TiDB Coprocessor Range Splitting Anomaly"
            message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} logged unexpected coprocessor key-range splitting anomaly (bucket/region boundary mismatch). Check TiDB logs for the matching keyword and fields (e.g. "reason").'