groups: - name: swat-errorlog-governance rules: - alert: PDLoadFromETCDMeetError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `etcdutil.go` |= `load from etcd meet error` [5m] )) ) > 9 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Loading etcd data is abnormal" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load from etcd meet error" If the error is not a network error, contact the oncall person directly' - alert: PDRevokeLeaseFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `lease.go` |= `revoke lease failed` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Affects tso renewal and pd leader election" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "revoke lease failed" When triggered, you can first check the disk related information. If there is no abnormality, notify the oncall personnel' - alert: PDGetTSError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `client.go` |= `[pd] getTS error` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Affect TSO service" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[pd] getTS error" If it is not a network error, you can contact oncall for troubleshooting' - alert: PDReportGetTSErrorInTSODispatcher expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `tso_dispatcher.go` |= `[tso] getTS error` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Affect TSO service" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[tso] getTS error" If it is not a network error, you can contact oncall for troubleshooting' - alert: PDGetTSErrorAfterProcessingRequests expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `tso_dispatcher.go` |= `[pd] getTS error` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Affect TSO service" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[pd] getTS error" If it is not a network error, you can contact oncall for troubleshooting' - alert: PDTsoRequestIsCanceledDueToTimeout expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , cluster_id!~"1379661944646416027", container="tidb"} |= `tso_dispatcher.go` |= `[tso] tso request is canceled due to timeout` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Affect TSO service" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[tso] tso request is canceled due to timeout" If it is not a network error, you can contact oncall for troubleshooting' - alert: PDUpdateTSError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `pd.go` |= `updateTS error` [1m] )) ) > 0 for: 2m labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "" message: '' - alert: TiDBAutoAnalyzeFailedInitializeQueue expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `refresher.go` |= `Failed to initialize the queue` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance annotations: summary: "Auto Analyze Queue Construction Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "Failed to initialize the queue" Unable to perform auto analyze for some time' - alert: TiDBRunDDLJobError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `[ddl] run DDL job error` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The DDL job execution failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[ddl] run DDL job error" Contact R&D to check the type and reason of the failed DDL' - alert: TiDBRunDDLJobModifyColumnNullError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633",component="tidb"} |= "DDL job submitted" |~ "(?i)modify\\s*column" |~ "(?i)null" [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The DDL job execution failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "DDL job submitted" with modify column null. Contact R&D to check the type and reason of the failed DDL. Please investigate by the SOP: https://pingcap.feishu.cn/wiki/WIFhwSKxOiupOWknXGrcjOEdn7c' - alert: TiDBAdminCheckFoundDataInconsistency expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `reporter.go` |= `admin check found data inconsistency` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance annotations: summary: "Data Accuracy Risk" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "admin check found data inconsistency" technical support required' - alert: TiDBAddIndexMayFailAndReportBuildIngestBackendFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `backend_mgr.go` |= `build ingest backend failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Add index registration backend failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "build ingest backend failed" which may cause failure to add index. Contact R&D to check the reason' - alert: TiDBDXFFailedAndReportCheckWhetherThereAreSubtasksFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `task_executor.go` |= `check whether there are subtasks to run failed` [3m] )) ) > 1 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The subtask executor failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "check whether there are subtasks to run failed" when checking whether there are subtasks that can be executed. This may cause add index or import into to fail. Contact R&D to check the reason' - alert: TiDBLoadDataFailedAndReportCommitErrorCheckAndInsert expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `load_data.go` |= `commit error CheckAndInsert` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Load data failed to import in batches" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "commit error CheckAndInsert" The disk may be full or the transaction may fail. Contact R&D to check the specific problem' - alert: TiDBLoadDataFailedAndReportCommitErrorRefresh expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `load_data.go` |= `commit error refresh` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The transaction submission of load data failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "commit error refresh" There may be duplicate keys and other problems. Contact R&D for inspection' - alert: TiDBRunSQLFailedAndReportCompileSQLPanic expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `compiler.go` |= `compile SQL panic` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Alarm requires specific investigation" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "compile SQL panic" find out why it panicked and then fixed' - alert: TiDBRunSQLMayFailAndReportConnectionRunningLoopPanic expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `conn.go` |= `connection running loop panic` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Alarm requires specific investigation" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "connection running loop panic" find out why it panicked and then fixed' - alert: TiDBDrainConnectionFailedInLoadData expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `conn.go` |= `drain connection failed in load data` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Load data When importing" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "drain connection failed in load data" an error occurs when reading a data packet from a connection. This may cause load data to fail. Contact R&D for inspection' - alert: TiDBDoImportFailedAndReportErrorOnEncodeAndSort expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `encode_and_sort_operator.go` |= `error on encode and sort` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Import into reports an error" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "error on encode and sort" during the encode & sort phase. Contact R&D to check the cause' - alert: TiDBReportErrorWhenGettingDdlHistoryCount expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `ddl.go` |= `error when getting the ddl history count` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Failure to obtain the DDL historical sequence number" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "error when getting the ddl history count" may cause problems such as abnormal owner status and abnormal DDL task scheduling. Contact R&D to check the cause' - alert: TiDBDXFFailedAndReportGetTaskFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `manager.go` |= `get task failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The distributed framework failed to query the task from the system table" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "get task failed" The subsequent task scheduling logic cannot continue to execute, which may cause the failure of adding indexes or importing into. Contact R&D to check the cause' - alert: TiDBAddIndedMayFailAndReportImportError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `backfilling_operators.go` |= `import error` != `Duplicate entry` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The data import into TikV fails" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "import error" when adding indexes. This may be caused by duplicate keys, etc. It is recommended to contact R&D to check the cause' - alert: TiDBAddIndexFailedAndReportIngestDataIntoStorageError expr: | max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `backend.go` |= `ingest data into storage error` [1m] ) ) > 0 unless sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `found duplicate key` [1m] ) ) > 0 ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Failure in importing indexed data into TikV" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "ingest data into storage error" may result in failure in indexing. It is recommended to contact R&D to check the cause. For specific reasons, see the error content in err id 76' - alert: TiDBLoadDataLocalFail expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `load_data.go` |= `load local data failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "load data local Failed to import data" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load local data failed" This may be due to problems such as the disk being full. Contact R&D to check the specific error content' - alert: TiDBInfoSchemReloadShemaFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `domain.go` |= `reload schema in loop failed` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "info schema Failed to load the latest schema information" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "reload schema in loop failed" Continuous failures may cause queries to use old version schemas and report errors. It is recommended that you contact R&D to check the specific cause.' - alert: TiDBShemaSyncerRestartFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `domain.go` |= `reload schema in loop, schema syncer restart failed` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "TiDB node synchronization schema version failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "reload schema in loop, schema syncer restart failed" the node may be unable to process DDL. It is recommended to contact R&D to check the etcd and network issues of the cluster' - alert: TiDBShemaSyncerRestartFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `domain.go` |= `restart the schema syncer failed` [1m] )) ) > 2 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The TiDB node failed to synchronize the schema version" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "restart the schema syncer failed" the node cannot process DDL. It is recommended to contact R&D to check the etcd and network issues of the cluster' - alert: TiDBDXFFailedAndReportTaskReverted expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `handle.go` |= `task reverted` != `Duplicate entry` != `Duplicate key` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The distributed framework task execution failed and rolled back" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "task reverted" Adding indexes or importing into on the user side may fail. For specific reasons, it is recommended to contact R&D to check the specific error information. It may be that adding uk encountered duplicate keys and other problems' - alert: TiFlashMemoryExceededLimit expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tiflash"} |= `SegmentReader.cpp` |= `ErrMsg: Memory limit (total) exceeded caused by 'RSS(Resident Set Size) much larger than limit'` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "Query failure caused by high memory pressure on TiFlash" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "ErrMsg: Memory limit (total) exceeded caused by RSS(Resident Set Size) much larger than limit" It may be that the query plan is wrong, or a new business query is added, or other problems cause the tiflash memory to be high. It is recommended to contact R&D for processing' - alert: TiFlashTLSHandshakeError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tiflash"} |= `status_server.rs` |= `Status server error: TLS handshake error` [1m] )) ) > 1 for: 4m labels: severity: critical response_class: page component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "" message: '' - alert: TiKVReadDefaultCFButDefaultValueNotFound expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tikv"} |= `mod.rs` |= `default value not found` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "Default CF data is missing when reading data" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "default value not found" Affects the correctness of business data and has the risk of panic. Check whether the cluster has lost data due to network or disk failures. If the problem cannot be solved, contact R&D' - alert: PDFailAddSchedulerWithIndependentConf expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `coordinator.go` |= `can not add scheduler with independent configuration` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Scheduler Cannot Be Initialized" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "can not add scheduler with independent configuration" The storage format may be incompatible. Contact R&D directly' - alert: PDRedirectButServerNotLeader expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `middleware.go` |= `redirect but server is not leader` [5m] )) ) > 9 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Http Request Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "redirect but server is not leader" are mainly ping and health requests, which are generally related to leader switching or TCP connection disconnection. If it is due to these two reasons, you can wait for self-recovery. If it is not due to either, it is recommended to contact R&D. If it is not recovered for more than 5 minutes, contact R&D. http requests have little impact on the business, and the core links are all grpc requests' - alert: PDGetTSError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `tso_dispatcher.go` |= `[tso] getTS error` [1m] )) ) > 9 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "TSO Request Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[tso] getTS error", which is usually related to leader switching or TCP connection disconnection. You can judge it through the log. If it is due to these two reasons, you can wait for self-recovery. If it is not an error, ask R&D support. If it is not recovered after more than 5 minutes, contact R&D' - alert: PDFailPassPrecheck expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `member.go` |= `failed to pass pre-check, check pd leader later` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Failed to expand the pd node" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to pass pre-check, check pd leader later" need to find R&D to troubleshoot' - alert: TiDBAutoAnalyzeFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", cluster_id!~"10652355502677513793|10795788448421792248", container="tidb"} |~ "(update\\.go|exec\\.go|autoanalyze\\.go)" |= `auto analyze failed` [5m] )) ) > 0 unless max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", cluster_id!~"10652355502677513793|10795788448421792248", container="tidb"} |= `parallel cpu profiler stopped` [5m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance annotations: summary: "TiDB auto analyze failed " message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "auto analyze failed". Please investigate by the SOP: https://pingcap.feishu.cn/wiki/Fe4vwRa4iiCgYRkg6wec6PkQnlh' - alert: TiDBFailLoadSafepointFromPD expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660|1372813089454544954", project_id != "", cluster_id!~"1379661944646416027", container="tidb"} |= `kv.go` |= `fail to load safepoint from pd` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance annotations: summary: "Failed to get information from pd" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "fail to load safepoint from pd" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support' - alert: TiDBFailLoadSafepointFromPD4wetechsaas01 expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", cluster_id=~"1379661944646416027", container="tidb"} |= `kv.go` |= `fail to load safepoint from pd` [1m] )) ) > 5 for: 1m labels: severity: major response_class: ticket component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance annotations: summary: "Failed to get information from pd" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "fail to load safepoint from pd" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support' - alert: TiDBDXFFailedAndReportFailedHandleTask expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `task_executor.go` |= `failed to handle task` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "Distributed framework handles errors when running tasks" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to handle task" add index or import into may fail. Contact R&D to check' - alert: TiDBDXFFailedAndReportGetSubtasksFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `task_executor.go` |= `get subtasks failed` !~ `context canceled` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The distributed framework reports an error" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "get subtasks failed" When the distributed framework is balancing subtask scheduling, an error occurs when reading the tidb_background_subtask system table. Add index or import into may fail. Contact R&D for inspection' - alert: TiDBLoadStoreFromPDFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |~ "(store_cache\\.go|region_cache\\.go)" |= `loadStore from PD failed` !~ `client connection is closing` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance annotations: summary: "Failed to get information from pd" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "loadStore from PD failed" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support' - alert: TiDBUnexpectedPanicInRecoverableGoroutine expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `misc.go` |= `panic in the recoverable goroutine` !~ "memory limit" !~ "Query execution was interrupted" [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "A panic occurred online" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "panic in the recoverable goroutine" Need to know the specific panic stack, then open an issue to resolve it' - alert: TiDBImportIntoReportErrPostProcessFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `subtask_executor.go` |= `post process failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "import into reports an error in the subtask during the post process phase" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "post process failed" The import into task may fail. Contact R&D for processing' - alert: TiDBDXFFailedAndReportRefreshTaskFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `task_executor.go` |= `refresh task failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "An error occurred while reading the tidb_global_task system table when checking task status" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "refresh task failed" It can add index or import into may fail, contact R&D to check' - alert: TiDBImportIntoReportErrRunSubtaskFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `task_executor.go` |= `run subtask failed` !~ `context canceled` != `Duplicate entry` != `Duplicate key` != `found duplicate key` != `subtask not found` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "import into reports an error in the post process phase" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "run subtask failed" The import into task may fail. Contact R&D for processing.' - alert: TiDBShuffleExecPanic expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `shuffle.go` |= `shuffle panicked` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-executor oncall_team: tidb_executor stability_governance: errorlog-governance annotations: summary: "BUG" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "shuffle panicked" please contact R&D for the work order' - alert: TiDBDXFFailedAndReportTaskExecutorMetFirstError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `task_executor.go` |= `taskExecutor met first error` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "The first error encountered when the distributed framework processes tasks" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "taskExecutor met first error" Import into or add index may fail. It is recommended to contact R&D to check the cause' - alert: TiKVTLSHandshakeError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} |= `mod.rs` |= `Status server error: TLS handshake error` [1m] )) ) > 1 for: 4m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "TLS handshake error" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "Status server error: TLS handshake error" Failed attempts to automatically repair the problem will result in the inability to connect to the node, making the node unavailable. It is recommended to check whether the network connection is smooth, whether the firewall blocks the communication, and the certificate authority. If the problem cannot be solved, contact the R&D personnel' - alert: TiKVExceedsRaftEntryMaxSize expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} |= `peer.rs` |= `entry is too large` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "The write log size exceeds raft-entry-max-size" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "entry is too large" After confirmation, consider whether to adjust the raft-entry-max-size parameter' - alert: TiKVFailedFetchSafepointFromPD expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} |= `gc_manager.rs` |= `failed to get safe point from pd` [1m] )) ) > 4 for: 1m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "Failed to obtain checkpoint from PD" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to get safe point from pd" Continuous occurrence may cause GC failure, resulting in redundant node space accumulation. This is usually caused by a problem with the PD communication. It is recommended to check the PD connection and status. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D' # - alert: TiKVFailedToSendSignificantRaftMsg # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} # |= `transport.rs` # |= `failed to send significant msg` # [1m] # )) # ) > 4 # for: 4m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # annotations: # summary: "Failed to send significant raft message" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to send significant msg" The transaction failure rate will increase. This is usually due to a TiKV node communication problem. It is recommended to check the network connection and node status. If it occurs for a short time, you can ignore it. If it occurs frequently for a long time, you need to be alerted. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D' - alert: TiVKFailedToWatchResourceGroups expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} |= `service.rs` |= `failed to send significant msg` [1m] )) ) > 1 for: 1m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "Unable to obtain PD information" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to send significant msg" If this happens for a long time, the node may become unavailable. This is usually caused by a problem with the PD communication. It is recommended to check the PD connection and status. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D' - alert: TiDBCannotGetIngestEngineInfo expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `backend.go` |= `can not get ingest engine info` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "TiDB Cannot Get Ingest Engine Info" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "can not get ingest engine info" errors during **add index ingest**. This may lead to **index addition failure**. Contact R&D for investigation and resolution.' # - alert: TiDBDoImportMeetsError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} # |= `local.go` # |= `do import meets error` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # annotations: # summary: "TiDB Do Import Meets Error" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "do import meets error". This indicates that a **Lightning import operation encountered an error**, possibly due to **duplicate keys** or other issues. Contact R&D for handling.' - alert: TiDBLoadStoreFromPDFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `region_cache.go` |= `loadStore from PD failed` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance annotations: summary: "TiDB Load Store From PD Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "loadStore from PD failed" errors in the last 1 minute. This failure to get information from PD could be due to **network issues or PD restarts**. If PD cannot communicate, the cluster might **become unavailable** and other alerts should also be present. When this occurs, **investigate network connectivity and PD load**, and observe TiDB''s pd-client monitoring. Monitor for self-recovery. No parameter adjustments are needed. If it does not self-recover, escalate to technical support.' - alert: TiDBDXFFailedAndReportTaskManagerMetError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `manager.go` |= `task manager met error` != `error="context canceled"` [5m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "TiDB Task Manager Met Error" message: 'TiDB cluster {{ $labels.cluster_id }} reported "task manager met error". This indicates a failure in the **distributed framework processing tasks**, potentially causing **add index and import into tasks to fail**. Contact R&D to check the root cause of the task or subtask failure.' - alert: TiDBUpdateCheckpointFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `checkpoint.go` |= `update checkpoint failed` [5m] )) ) > 20 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance annotations: summary: "TiDB Update Checkpoint Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "update checkpoint failed" errors. This occurs during **add index ingest**, preventing **ingest progress** and causing **add index failures**. Contact R&D to investigate the cause of the failure.' - alert: TiDBUpdateTSError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `pd.go` |= `updateTS error` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance annotations: summary: "TiDB Update TSO Error" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "updateTS error" in the last 1 minute. This failure to get information from PD could be due to **network issues or PD restarts**. If PD cannot communicate, the cluster might **become unavailable** and other alerts should also be present. When this occurs, **investigate network connectivity and PD load**, and observe TiDB''s pd-client monitoring. Monitor for self-recovery. No parameter adjustments are needed. If it does not self-recover, escalate to technical support.' - alert: TiDBPlannerHandleDdlEventFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `domain.go` |= `handle ddl event failed` [5m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance annotations: summary: "TiDB Planner Handle DDL Event Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "handle ddl event failed" errors more than 4 times in the last 5 minutes. Currently, this error **is unlikely to affect business operations**. If the cluster load is high (e.g., increased query backoff), this error can be ignored. However, if the cluster load is low, this error is not expected by design and **requires R&D intervention**.' - alert: TiFlashRowKeyMetUnexpectedEncodingFormat expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tiflash"} |= `RowKeyRange.h` |= `unexpected encoding format` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "TiFlash RowKey Met Unexpected Encoding Format" message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} detected "unexpected encoding format" in RowKey. This indicates **data inconsistency** caused by an irregular key. This alert ensures that even if `profiles.default.dt_enable_ingest_check` is set to `false`, data inconsistency issues due to irregular keys are detected via logs instead of causing a TiFlash panic. An SOP can then be followed to **retrieve logs and identify affected regions for manual data inconsistency repair**. **Contact R&D directly** to handle this alert.' - alert: TiFlashMonotonicTimeJumpedBack expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tiflash"} |= `time.rs` |= `monotonic time jumped back` [1m] )) ) > 1 for: 1m labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "TiFlash Monotonic Time Jumped Back" message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "monotonic time jumped back" errors more than 1 time in the last 5 minutes. This signifies a **clock jump (time callback)**, which is **severe** and could potentially lead to **cluster crashes**. **Contact R&D directly** upon this alert.' - alert: TiFlashResolveStoreAddressFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tiflash"} |= `raft_client.rs` |= `resolve store address failed` !~ "has been removed" [1m] )) ) > 4 for: 1m labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "TiFlash Resolve Store Address Failed" message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "resolve store address failed" errors more than 4 times in the last 1 minute. The inability to resolve store addresses, if persistent, will **lead to KV node failures**. This could be due to **disk offline events** or **old metadata not being properly cleaned up** after a disk was decommissioned. First, check if the disk is offline. If the disk is mounted and the error log shows a `store_id` mismatch, try using `tikv_ctl remove-fail-stores` or `pd_ctl store delete` to remove the old `store_id` information. If unresolved, **contact R&D**.' - alert: TiKVIngestFail expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} |= `apply.rs` |= `ingest fail` [1m] )) ) > 4 for: 1m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "TiKV Ingest Fail" message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "ingest fail" errors more than 4 times in the last 1 minute. This indicates **data ingestion failure**, typically due to **region version mismatches**. Occasional occurrences may self-recover. Frequent occurrences could **impact data import**. If this issue arises, first **check the node''s status**, including load and network communication. If it does not self-recover, **examine the specific error type in the logs and contact R&D**.' - alert: TiKVResolveStoreAddressFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} |= `raft_client.rs` |= `resolve store address failed` !~ "StoreTombstone" [1m] )) ) > 4 for: 1m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "TiKV Resolve Store Address Failed" message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "resolve store address failed" errors more than 4 times in the last 1 minute. The inability to resolve store addresses, if persistent, will **lead to KV node failures**. This could be due to **disk offline events** or **old metadata not being properly cleaned up** after a disk was decommissioned. First, check if the disk is offline. If the disk is mounted and the error log shows a `store_id` mismatch, try using `tikv_ctl remove-fail-stores` or `pd_ctl store delete` to remove the old `store_id` information. If unresolved, **contact R&D**.' - alert: TiDBMetExpensiveQuery expr: | max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `[expensive_query]` != `sql="analyze` != `sql="ANALYZE` [10m] ) ) ) > 50 and max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} |= `tp add index` |= `Type:modify column` [10m] ) ) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance annotations: summary: "Too much expensive queries" message: 'TiDB cluster {{ $labels.cluster_id }} just met too much expensive queries' - alert: TiDBExpensiveQueryCntGt100For1m expr: | max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635", project_id=~"1372813089454544954", project_id != "" , container="tidb"} |= `[expensive_query]` != `sql="analyze` != `sql="ANALYZE` [10m] ) ) ) > 100 for: 1m labels: severity: major response_class: ticket component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance annotations: summary: "A large number of running expensive queries exceeding 60 seconds." message: 'TiDB cluster {{ $labels.cluster_id }} just met 100+ expensive queries exceeding 1 minute.' - alert: TiDBExpensiveQueryCntGt500AndIncreaseFor10m expr: | (max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635", project_id=~"1372813089454544954", project_id != "" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [10m] ) ) ) > 500) and (max by (cluster_id, project_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635", project_id=~"1372813089454544954", project_id != "" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [1m] ) ) > 3 * max by (cluster_id, project_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635", project_id=~"1372813089454544954", project_id != "" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [1m] offset 10m ) )) for: 0s labels: severity: critical response_class: page component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance annotations: summary: "A large number of running expensive queries exceeding 10 min." message: 'TiDB cluster {{ $labels.cluster_id }} just met 500+ running expensive queries and is 3x higher than 10min before.' - alert: PDFailedToPersistStore expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `cluster.go` |= `failed to persist store` [1m] )) ) > 4 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "PD Failed to Persist Store" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to persist store" errors more than 4 times in the last 1 minute. **This significantly impacts TSO (Timestamp Oracle) and heartbeat mechanisms.** It''s often due to **disk issues**. We recommend checking the **disk status** using Node Exporter.' - alert: PDLoadFromEtcdError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `etcdutil.go` |= `load from etcd meet error` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "PD Load From Etcd Error" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load from etcd meet error" errors more than 4 times in the last 1 minute. This indicates PD cannot load data stored in Etcd, leading to **scheduler and TSO issues**. Check **disk read/write latency** via Node Exporter. If it''s not a latency problem, escalate to R&D.' - alert: PDGetTSError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `tso_dispatcher.go` |= `getTS error` [5m] )) ) > 9 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "PD Get TSO Error" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "getTS error" more than 9 times in the last 5 minutes. This means user TSO requests are failing, causing **SQL execution failures**. It''s usually related to **leader switching or TCP connection disconnections**. These issues often **self-recover**; verify via logs. If not one of these reasons, request R&D support.' - alert: PDFailedToCloseGRPC expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `client.go` |= `failed to close gRPC` [5m] )) ) > 9 for: 0s labels: severity: warning component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "PD Failed to Close gRPC Connection" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to close gRPC" errors more than 9 times in the last 5 minutes. This indicates a **connection leak**, which will **consume some CPU resources**. If this persists for 5 minutes, engage R&D for investigation.' - alert: PDSaveToEtcdError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `etcd_kv.go` |= `save to etcd meet error` [1m] )) ) > 9 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "PD Save to Etcd Error" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "save to etcd meet error" errors more than 9 times in the last 1 minute. This may **affect TSO service and prevent SQL execution**. First, check **disk read/write conditions** via Node Exporter. If normal, escalate to R&D.' - alert: RUTokenStateAnomalyInvalidAssignedToken expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `token_buckets.go` |= `assigned token is invalid` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "RU Token State Anomaly - Invalid Assigned Token" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "assigned token is invalid"' - alert: RUTokenStateAnomalyInvalidTokenSlot expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="pd"} |= `token_buckets.go` |= `slot token capacity is invalid` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "RU Token State Anomaly - Invalid Token Slot" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "slot token capacity is invalid"' - alert: AbnormalJoinedMember expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `join.go` |= `there is an abnormal joined member in the current member list` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Abnormal Joined Member" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "there is an abnormal joined member in the current member list" The configuration is incorrect, there is no corresponding setting name' - alert: PersistMinResolvedTSMeetError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="pd"} |= `cluster.go` |= `persist min resolved ts meet error` [5m] )) ) > 9 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance annotations: summary: "Persist Min Resolved TS Meet Error" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "persist min resolved ts meet error" etcd storage failure, usually a storage problem' - alert: RebuildTableAnalysisJobQueueFail expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `autoanalyze.go` |= `rebuild table analysis job queue failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Rebuild Table Analysis Job Queue Fail" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "rebuild table analysis job queue failed" The situation of excessive tikv pressure such as tikv unavailable can be ignored, and other clearing alarms will be sent directly. But this is not a serious problem' - alert: LightningImportFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `backend.go` |= `import failed` !~ `context canceled` != `found duplicate key` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Lightning Import Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "import failed" Lightning import failed. Unless the user canceled it, please contact R&D to check the specific reason.' - alert: DDLCannotFindReorgCtx expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `ddl.go` |= `cannot find reorgCtx` [1m] )) ) > 0 for: 3m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "DDL Cannot Find Reorg Ctx" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "cannot find reorgCtx" reorgCtx is used to notify the reorg worker of the status change when the DDL add index or modify column rollback occurs. Continuous occurrence of this error may cause the add index or modify column rollback to fail.' - alert: FailedToUploadSafepointToPD expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `gc_worker.go` |= `failed to upload safe point to PD` [1m] )) ) > 0 for: 2m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Failed To Upload Safepoint To PD" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to upload safe point to PD"' - alert: LoadDataErrorAndReportCommitWorkPanicked expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `load_data.go` |= `commitWork panicked` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Load Data Error And Report Commit Work Panicked" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "commitWork panicked" When loading data, inserting data fails because the memory quota is exceeded. You can try temporarily increasing the mem limit or reducing the tidb_dml_batch_size and try again.' - alert: ImportIntoOrAddIndexReportScanRegionFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `local.go` |= `scan region failed` !~ `context canceled` [1m] )) ) > 0 for: 1m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Import Into Or Add Index Report Scan Region Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "scan region failed"' - alert: RegionCacheLoadRegionFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `region_cache.go` |= `load region failure` [1m] )) ) > 0 for: 2m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Region Cache Load Region Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load region failure"' - alert: ValidateReadTSFailedForRequest expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `region_request.go` |= `validate read ts failed for request` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Validate Read TS Failed For Request" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "validate read ts failed for request"' - alert: RegionUnavailable expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tidb"} |= `txn.go` |= `region unavailable` [1m] )) ) > 0 for: 2m labels: severity: major response_class: ticket component: tidb oncall_team: resilience stability_governance: errorlog-governance annotations: summary: "Region Unavailable" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "region unavailable"' - alert: TiFlashRSSMuchLargerThanLimit expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tiflash"} |= `DAGDriver.cpp` |= `[FLASH:Coprocessor:MemoryLimitExceeded] Memory limit (total) exceeded caused by 'RSS(Resident Set Size) much larger than limit'` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "TiFlash RSS Much Larger Than Limit" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[FLASH:Coprocessor:MemoryLimitExceeded] Memory limit (total) exceeded caused by "RSS(Resident Set Size) much larger than limit" The query failed due to high TiFlash memory pressure. This could be due to an incorrect query plan, a newly added business query, or other issues causing high TiFlash memory usage. We recommend contacting our oncall team for assistance.' - alert: TiFlashRSSMuchLargerThanLimit expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tiflash"} |= `Exception.cpp` |= `RSS(Resident Set Size) much larger than limit` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "TiFlash RSS Much Larger Than Limit" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "RSS(Resident Set Size) much larger than limit" The query failed due to high TiFlash memory pressure. This could be due to an incorrect query plan, a newly added business query, or other issues causing high TiFlash memory usage. We recommend contacting our oncall team for assistance.' - alert: TiFlashSyncSchemaFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"This error is caused by TiFlash's inability to connect to PD during the process of deletion of the cluster, and it does not need to be processed.", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tiflash"} |= `SchemaSyncService.cpp` |= `RSS(Resident Set Size) much larger than limit` [1m] )) ) > 9 for: 0s labels: severity: major response_class: ticket component: tiflash oncall_team: tiflash stability_governance: errorlog-governance annotations: summary: "TiFlash Sync Schema Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "RSS(Resident Set Size) much larger than limit" Broadly speaking, this is an error caused by the inability to connect to the PD.' - alert: BRBackupCreateStorageFailed expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", project_id != "" , container="tikv"} |= `endpoint.rs` |= `backup create storage failed` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance annotations: summary: "BRBackup Create Storage Failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "backup create storage failed"' # - alert: TiDBPlannerDumpStatsDeltaFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} # |= `handle.go` # |= `dump stats delta fail` # [5m] # )) # ) > 1 # for: 0s # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # annotations: # summary: "TiDB Planner Dump Stats Delta Failed" # message: "TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported 'dump stats delta fail' error logs more than 1 time in the last 5 minutes. This will **affect auto-analyze or row count estimation**, consequently impacting **execution plan selection**. If the cluster load is high (e.g., increased query backoff), this error can be ignored. If the cluster load is low, this error is not expected and **requires R&D intervention**." # - alert: TiKVLeaderClientFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} # |= `deadlock.rs` # |= `leader client failed` # [1m] # )) # ) > 0 # for: 5m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # annotations: # summary: "TiKV Leader Client Failed. Please check whether the cluster restart first." # message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported a "leader client failed" error. This means TiKV cannot send messages to the **deadlock leader TiKV node**, potentially preventing TiKV from correctly identifying transaction deadlocks. **Verify network connectivity** and check if any TiKV nodes are down. If only this alert appears without other network-related alerts, **escalate to technical support**.' # - alert: TiKVMonotonicTimeJumpedBack # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} # |= `time.rs` # |= `monotonic time jumped back` # [1m] # )) # ) > 1 # for: 1m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # annotations: # summary: "TiKV Monotonic Time Jumped Back" # message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "monotonic time jumped back" errors more than 1 time in the last 5 minutes. This signifies a **clock jump (time callback)**, which is **severe** and could potentially lead to **cluster crashes**. **Contact R&D directly** upon this alert.' # - alert: TiDBPlannerHandleOneItemTaskPanicked # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tidb"} # |= `handle_hist.go` # |= `handleOneItemTask panicked` # [2m] # )) # ) > 1 # for: 0s # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # annotations: # summary: "TiDB Planner Handle One Item Task Panicked" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "handleOneItemTask panicked" errors more than 1 time in the last 2 minutes. This indicates an issue with **synchronously loading statistics during SQL execution**, which might lead to **suboptimal execution plans**. **R&D intervention is required** for investigation.' # - alert: TiKVAsyncWriteOnAppliedCallbackDropped # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454576822|1372813089454574660", project_id != "", container="tikv"} # |= `mod.rs` # |= `async write on_applied callback is dropped` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # annotations: # summary: "TiKV async write on applied callback dropped" # message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "TiKV async write on applied callback dropped. After a request is applied but there is no callback, data inconsistency may occur.Try restarting the affected TiKV node; if the issue persists, please contact the development team.'