groups: - name: swat-errorlog-japan rules: # - alert: PDLoadFromETCDMeetError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `etcdutil.go` # |= `load from etcd meet error` # [5m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Loading etcd data is abnormal" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load from etcd meet error" If the error is not a network error, contact the oncall person directly' # - alert: PDRevokeLeaseFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `lease.go` # |= `revoke lease failed` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Affects tso renewal and pd leader election" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "revoke lease failed" When triggered, you can first check the disk related information. If there is no abnormality, notify the oncall personnel' # - alert: PDGetTSError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `client.go` # |= `[pd] getTS error` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Affect TSO service" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[pd] getTS error" If it is not a network error, you can contact oncall for troubleshooting' # - alert: PDReportGetTSErrorInTSODispatcher # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `tso_dispatcher.go` # |= `[tso] getTS error` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Affect TSO service" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[tso] getTS error" If it is not a network error, you can contact oncall for troubleshooting' # - alert: PDGetTSErrorAfterProcessingRequests # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `tso_dispatcher.go` # |= `[pd] getTS error` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Affect TSO service" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[pd] getTS error" If it is not a network error, you can contact oncall for troubleshooting' # - alert: PDTsoRequestIsCanceledDueToTimeout # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `tso_dispatcher.go` # |= `[tso] tso request is canceled due to timeout` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Affect TSO service" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[tso] tso request is canceled due to timeout" If it is not a network error, you can contact oncall for troubleshooting' # - alert: PDUpdateTSError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `pd.go` # |= `updateTS error` # [1m] # )) # ) > 0 # for: 2m # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "" # message: '' # - alert: TiDBAutoAnalyzeFailedInitializeQueue # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `refresher.go` # |= `Failed to initialize the queue` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Auto Analyze Queue Construction Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "Failed to initialize the queue" Unable to perform auto analyze for some time' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiDBRunDDLJobError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} |= `[ddl] run DDL job error` [1m] )) ) > 0 for: 0s labels: severity: major response_class: ticket component: tidb-sql-data oncall_team: tidb_sql stability_governance: errorlog-governance regions: japan annotations: summary: "The DDL job execution failed" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[ddl] run DDL job error" Contact R&D to check the type and reason of the failed DDL' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiDBAdminCheckFoundDataInconsistency expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} |= `reporter.go` |= `admin check found data inconsistency` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tidb-txn oncall_team: tidb_txn stability_governance: errorlog-governance regions: japan annotations: summary: "Data Accuracy Risk" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "admin check found data inconsistency" technical support required' # - alert: TiDBAddIndexMayFailAndReportBuildIngestBackendFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `backend_mgr.go` # |= `build ingest backend failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Add index registration backend failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "build ingest backend failed" which may cause failure to add index. Contact R&D to check the reason' # - alert: TiDBDXFFailedAndReportCheckWhetherThereAreSubtasksFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `task_executor.go` # |= `check whether there are subtasks to run failed` # [3m] # )) # ) > 1 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The subtask executor failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "check whether there are subtasks to run failed" when checking whether there are subtasks that can be executed. This may cause add index or import into to fail. Contact R&D to check the reason' # - alert: TiDBLoadDataFailedAndReportCommitErrorCheckAndInsert # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `load_data.go` # |= `commit error CheckAndInsert` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Load data failed to import in batches" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "commit error CheckAndInsert" The disk may be full or the transaction may fail. Contact R&D to check the specific problem' # - alert: TiDBLoadDataFailedAndReportCommitErrorRefresh # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `load_data.go` # |= `commit error refresh` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The transaction submission of load data failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "commit error refresh" There may be duplicate keys and other problems. Contact R&D for inspection' # - alert: TiDBRunSQLFailedAndReportCompileSQLPanic # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `compiler.go` # |= `compile SQL panic` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Alarm requires specific investigation" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "compile SQL panic" find out why it panicked and then fixed' # - alert: TiDBRunSQLMayFailAndReportConnectionRunningLoopPanic # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `conn.go` # |= `connection running loop panic` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Alarm requires specific investigation" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "connection running loop panic" find out why it panicked and then fixed' # - alert: TiDBDrainConnectionFailedInLoadData # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `conn.go` # |= `drain connection failed in load data` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Load data When importing" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "drain connection failed in load data" an error occurs when reading a data packet from a connection. This may cause load data to fail. Contact R&D for inspection' # - alert: TiDBDoImportFailedAndReportErrorOnEncodeAndSort # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `encode_and_sort_operator.go` # |= `error on encode and sort` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Import into reports an error" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "error on encode and sort" during the encode & sort phase. Contact R&D to check the cause' # - alert: TiDBReportErrorWhenGettingDdlHistoryCount # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `ddl.go` # |= `error when getting the ddl history count` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failure to obtain the DDL historical sequence number" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "error when getting the ddl history count" may cause problems such as abnormal owner status and abnormal DDL task scheduling. Contact R&D to check the cause' # - alert: TiDBDXFFailedAndReportGetTaskFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `manager.go` # |= `get task failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The distributed framework failed to query the task from the system table" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "get task failed" The subsequent task scheduling logic cannot continue to execute, which may cause the failure of adding indexes or importing into. Contact R&D to check the cause' # - alert: TiDBAddIndedMayFailAndReportImportError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `backfilling_operators.go` # |= `import error` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The data import into TikV fails" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "import error" when adding indexes. This may be caused by duplicate keys, etc. It is recommended to contact R&D to check the cause' # - alert: TiDBAddIndexFailedAndReportIngestDataIntoStorageError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `backend.go` # |= `ingest data into storage error` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failure in importing indexed data into TikV" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "ingest data into storage error" may result in failure in indexing. It is recommended to contact R&D to check the cause. For specific reasons, see the error content in err id 76' # - alert: TiDBLoadDataLocalFail # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `load_data.go` # |= `load local data failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "load data local Failed to import data" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load local data failed" This may be due to problems such as the disk being full. Contact R&D to check the specific error content' # - alert: TiDBInfoSchemReloadShemaFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `domain.go` # |= `reload schema in loop failed` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "info schema Failed to load the latest schema information" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "reload schema in loop failed" Continuous failures may cause queries to use old version schemas and report errors. It is recommended that you contact R&D to check the specific cause.' # - alert: TiDBShemaSyncerRestartFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `domain.go` # |= `reload schema in loop, schema syncer restart failed` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB node synchronization schema version failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "reload schema in loop, schema syncer restart failed" the node may be unable to process DDL. It is recommended to contact R&D to check the etcd and network issues of the cluster' # - alert: TiDBShemaSyncerRestartFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `domain.go` # |= `restart the schema syncer failed` # [1m] # )) # ) > 2 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The TiDB node failed to synchronize the schema version" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "restart the schema syncer failed" the node cannot process DDL. It is recommended to contact R&D to check the etcd and network issues of the cluster' # - alert: TiDBDXFFailedAndReportTaskReverted # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `handle.go` # |= `task reverted` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The distributed framework task execution failed and rolled back" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "task reverted" Adding indexes or importing into on the user side may fail. For specific reasons, it is recommended to contact R&D to check the specific error information. It may be that adding uk encountered duplicate keys and other problems' # - alert: TiFlashMemoryExceededLimit # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `SegmentReader.cpp` # |= `ErrMsg: Memory limit (total) exceeded caused by 'RSS(Resident Set Size) much larger than limit'` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Query failure caused by high memory pressure on TiFlash" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "ErrMsg: Memory limit (total) exceeded caused by RSS(Resident Set Size) much larger than limit" It may be that the query plan is wrong, or a new business query is added, or other problems cause the tiflash memory to be high. It is recommended to contact R&D for processing' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiFlashTLSHandshakeError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} |= `status_server.rs` |= `Status server error: TLS handshake error` [1m] )) ) > 1 for: 4m labels: severity: critical response_class: page component: tiflash oncall_team: tiflash stability_governance: errorlog-governance regions: japan annotations: summary: "" message: '' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiKVReadDefaultCFButDefaultValueNotFound expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} |= `mod.rs` |= `default value not found` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance regions: japan annotations: summary: "Default CF data is missing when reading data" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "default value not found" Affects the correctness of business data and has the risk of panic. Check whether the cluster has lost data due to network or disk failures. If the problem cannot be solved, contact R&D' # - alert: PDFailAddSchedulerWithIndependentConf # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `coordinator.go` # |= `can not add scheduler with independent configuration` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Scheduler Cannot Be Initialized" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "can not add scheduler with independent configuration" The storage format may be incompatible. Contact R&D directly' # - alert: PDRedirectButServerNotLeader # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `middleware.go` # |= `redirect but server is not leader` # [5m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Http Request Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "redirect but server is not leader" are mainly ping and health requests, which are generally related to leader switching or TCP connection disconnection. If it is due to these two reasons, you can wait for self-recovery. If it is not due to either, it is recommended to contact R&D. If it is not recovered for more than 5 minutes, contact R&D. http requests have little impact on the business, and the core links are all grpc requests' # - alert: PDGetTSError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `tso_dispatcher.go` # |= `[tso] getTS error` # [1m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TSO Request Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[tso] getTS error", which is usually related to leader switching or TCP connection disconnection. You can judge it through the log. If it is due to these two reasons, you can wait for self-recovery. If it is not an error, ask R&D support. If it is not recovered after more than 5 minutes, contact R&D' # - alert: PDFailPassPrecheck # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `member.go` # |= `failed to pass pre-check, check pd leader later` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failed to expand the pd node" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to pass pre-check, check pd leader later" need to find R&D to troubleshoot' # - alert: TiDBAutoAnalyzeFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id!~"10646585945675031829|10961033529265313143|10016617196374558903|1379661944646416090|1379661944637854312|10295689775695492430|10947714666900611238", container="tidb"} # |~ "(update\\.go|exec\\.go|autoanalyze\\.go)" # |= `auto analyze failed` # [5m] # )) # ) > 0 # unless # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `parallel cpu profiler stopped` # [5m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB auto analyze failed " # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "auto analyze failed". Please investigate by the SOP: https://pingcap.feishu.cn/wiki/Fe4vwRa4iiCgYRkg6wec6PkQnlh' # - alert: TiDBFailLoadSafepointFromPD # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `kv.go` # |= `fail to load safepoint from pd` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb-txn # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failed to get information from pd" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "fail to load safepoint from pd" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support' # - alert: TiDBDXFFailedAndReportFailedHandleTask # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `task_executor.go` # |= `failed to handle task` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Distributed framework handles errors when running tasks" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to handle task" add index or import into may fail. Contact R&D to check' # - alert: TiDBDXFFailedAndReportGetSubtasksFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `task_executor.go` # |= `get subtasks failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The distributed framework reports an error" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "get subtasks failed" When the distributed framework is balancing subtask scheduling, an error occurs when reading the tidb_background_subtask system table. Add index or import into may fail. Contact R&D for inspection' # - alert: TiDBLoadStoreFromPDFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |~ "(store_cache\\.go|region_cache\\.go)" # |= `loadStore from PD failed` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb-txn # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failed to get information from pd" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "loadStore from PD failed" It may be a network problem or pd restart. If pd cannot communicate, the cluster may not be able to provide services, and other alarms should be accompanied. When it occurs, you need to check the network and pd load, and observe the pd-client monitoring of tidb. Observe whether it recovers. No need to adjust parameters. If it cannot recover by itself, you need to escalate to technical support' # - alert: TiDBUnexpectedPanicInRecoverableGoroutine # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `misc.go` # |= `panic in the recoverable goroutine` # !~ "memory limit" # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "A panic occurred online" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "panic in the recoverable goroutine" Need to know the specific panic stack, then open an issue to resolve it' # - alert: TiDBImportIntoReportErrPostProcessFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `subtask_executor.go` # |= `post process failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "import into reports an error in the subtask during the post process phase" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "post process failed" The import into task may fail. Contact R&D for processing' # - alert: TiDBDXFFailedAndReportRefreshTaskFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `task_executor.go` # |= `refresh task failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "An error occurred while reading the tidb_global_task system table when checking task status" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "refresh task failed" It can add index or import into may fail, contact R&D to check' # - alert: TiDBImportIntoReportErrRunSubtaskFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `task_executor.go` # |= `run subtask failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "import into reports an error in the post process phase" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "run subtask failed" The import into task may fail. Contact R&D for processing.' # - alert: TiDBShuffleExecPanic # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `shuffle.go` # |= `shuffle panicked` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-executor # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "BUG" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "shuffle panicked" please contact R&D for the work order' # - alert: TiDBDXFFailedAndReportTaskExecutorMetFirstError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `task_executor.go` # |= `taskExecutor met first error` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "The first error encountered when the distributed framework processes tasks" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "taskExecutor met first error" Import into or add index may fail. It is recommended to contact R&D to check the cause' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiKVTLSHandshakeError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} |= `mod.rs` |= `Status server error: TLS handshake error` [1m] )) ) > 1 for: 4m labels: severity: major response_class: ticket component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance regions: japan annotations: summary: "TLS handshake error" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "Status server error: TLS handshake error" Failed attempts to automatically repair the problem will result in the inability to connect to the node, making the node unavailable. It is recommended to check whether the network connection is smooth, whether the firewall blocks the communication, and the certificate authority. If the problem cannot be solved, contact the R&D personnel' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiKVExceedsRaftEntryMaxSize expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} |= `peer.rs` |= `entry is too large` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tikv oncall_team: tikv_unified stability_governance: errorlog-governance regions: japan annotations: summary: "The write log size exceeds raft-entry-max-size" message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "entry is too large" After confirmation, consider whether to adjust the raft-entry-max-size parameter' # - alert: TiKVFailedFetchSafepointFromPD # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} # |= `gc_manager.rs` # |= `failed to get safe point from pd` # [1m] # )) # ) > 4 # for: 1m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failed to obtain checkpoint from PD" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to get safe point from pd" Continuous occurrence may cause GC failure, resulting in redundant node space accumulation. This is usually caused by a problem with the PD communication. It is recommended to check the PD connection and status. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D' # - alert: TiKVFailedToSendSignificantRaftMsg # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} # |= `transport.rs` # |= `failed to send significant msg` # [1m] # )) # ) > 4 # for: 4m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failed to send significant raft message" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to send significant msg" The transaction failure rate will increase. This is usually due to a TiKV node communication problem. It is recommended to check the network connection and node status. If it occurs for a short time, you can ignore it. If it occurs frequently for a long time, you need to be alerted. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D' # - alert: TiVKFailedToWatchResourceGroups # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} # |= `service.rs` # |= `failed to send significant msg` # [1m] # )) # ) > 1 # for: 1m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Unable to obtain PD information" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to send significant msg" If this happens for a long time, the node may become unavailable. This is usually caused by a problem with the PD communication. It is recommended to check the PD connection and status. If it is not caused by a network problem, you need to check the specific error type in the log and contact R&D' # - alert: TiDBCannotGetIngestEngineInfo # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `backend.go` # |= `can not get ingest engine info` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB Cannot Get Ingest Engine Info" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "can not get ingest engine info" errors during **add index ingest**. This may lead to **index addition failure**. Contact R&D for investigation and resolution.' # - alert: TiDBDoImportMeetsError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", container="tidb"} # |= `local.go` # |= `do import meets error` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # annotations: # summary: "TiDB Do Import Meets Error" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "do import meets error". This indicates that a **Lightning import operation encountered an error**, possibly due to **duplicate keys** or other issues. Contact R&D for handling.' # - alert: TiDBLoadStoreFromPDFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `region_cache.go` # |= `loadStore from PD failed` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb-txn # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB Load Store From PD Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "loadStore from PD failed" errors in the last 1 minute. This failure to get information from PD could be due to **network issues or PD restarts**. If PD cannot communicate, the cluster might **become unavailable** and other alerts should also be present. When this occurs, **investigate network connectivity and PD load**, and observe TiDB''s pd-client monitoring. Monitor for self-recovery. No parameter adjustments are needed. If it does not self-recover, escalate to technical support.' # - alert: TiDBDXFFailedAndReportTaskManagerMetError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `manager.go` # |= `task manager met error` # != `error="context canceled"` # [5m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB Task Manager Met Error" # message: 'TiDB cluster {{ $labels.cluster_id }} reported "task manager met error". This indicates a failure in the **distributed framework processing tasks**, potentially causing **add index and import into tasks to fail**. Contact R&D to check the root cause of the task or subtask failure.' # - alert: TiDBUpdateCheckpointFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `checkpoint.go` # |= `update checkpoint failed` # [5m] # )) # ) > 20 # for: 0s # labels: # severity: major # component: tidb-sql-data # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB Update Checkpoint Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "update checkpoint failed" errors. This occurs during **add index ingest**, preventing **ingest progress** and causing **add index failures**. Contact R&D to investigate the cause of the failure.' # - alert: TiDBUpdateTSError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `pd.go` # |= `updateTS error` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb-txn # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB Update TSO Error" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "updateTS error" in the last 1 minute. This failure to get information from PD could be due to **network issues or PD restarts**. If PD cannot communicate, the cluster might **become unavailable** and other alerts should also be present. When this occurs, **investigate network connectivity and PD load**, and observe TiDB''s pd-client monitoring. Monitor for self-recovery. No parameter adjustments are needed. If it does not self-recover, escalate to technical support.' # - alert: TiDBPlannerHandleDdlEventFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `domain.go` # |= `handle ddl event failed` # [5m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiDB Planner Handle DDL Event Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "handle ddl event failed" errors more than 4 times in the last 5 minutes. Currently, this error **is unlikely to affect business operations**. If the cluster load is high (e.g., increased query backoff), this error can be ignored. However, if the cluster load is low, this error is not expected by design and **requires R&D intervention**.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiFlashRowKeyMetUnexpectedEncodingFormat expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} |= `RowKeyRange.h` |= `unexpected encoding format` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tiflash oncall_team: tiflash stability_governance: errorlog-governance regions: japan annotations: summary: "TiFlash RowKey Met Unexpected Encoding Format" message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} detected "unexpected encoding format" in RowKey. This indicates **data inconsistency** caused by an irregular key. This alert ensures that even if `profiles.default.dt_enable_ingest_check` is set to `false`, data inconsistency issues due to irregular keys are detected via logs instead of causing a TiFlash panic. An SOP can then be followed to **retrieve logs and identify affected regions for manual data inconsistency repair**. **Contact R&D directly** to handle this alert.' # - alert: TiFlashMonotonicTimeJumpedBack # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `time.rs` # |= `monotonic time jumped back` # [1m] # )) # ) > 1 # for: 1m # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiFlash Monotonic Time Jumped Back" # message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "monotonic time jumped back" errors more than 1 time in the last 5 minutes. This signifies a **clock jump (time callback)**, which is **severe** and could potentially lead to **cluster crashes**. **Contact R&D directly** upon this alert.' # - alert: TiFlashResolveStoreAddressFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `raft_client.rs` # |= `resolve store address failed` # !~ "has been removed" # [1m] # )) # ) > 4 # for: 1m # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiFlash Resolve Store Address Failed" # message: 'TiFlash instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "resolve store address failed" errors more than 4 times in the last 1 minute. The inability to resolve store addresses, if persistent, will **lead to KV node failures**. This could be due to **disk offline events** or **old metadata not being properly cleaned up** after a disk was decommissioned. First, check if the disk is offline. If the disk is mounted and the error log shows a `store_id` mismatch, try using `tikv_ctl remove-fail-stores` or `pd_ctl store delete` to remove the old `store_id` information. If unresolved, **contact R&D**.' # - alert: TiKVIngestFail # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} # |= `apply.rs` # |= `ingest fail` # [1m] # )) # ) > 4 # for: 1m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiKV Ingest Fail" # message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "ingest fail" errors more than 4 times in the last 1 minute. This indicates **data ingestion failure**, typically due to **region version mismatches**. Occasional occurrences may self-recover. Frequent occurrences could **impact data import**. If this issue arises, first **check the node''s status**, including load and network communication. If it does not self-recover, **examine the specific error type in the logs and contact R&D**.' # - alert: TiKVResolveStoreAddressFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} # |= `raft_client.rs` # |= `resolve store address failed` # !~ "StoreTombstone" # [1m] # )) # ) > 4 # for: 1m # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiKV Resolve Store Address Failed" # message: 'TiKV instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "resolve store address failed" errors more than 4 times in the last 1 minute. The inability to resolve store addresses, if persistent, will **lead to KV node failures**. This could be due to **disk offline events** or **old metadata not being properly cleaned up** after a disk was decommissioned. First, check if the disk is offline. If the disk is mounted and the error log shows a `store_id` mismatch, try using `tikv_ctl remove-fail-stores` or `pd_ctl store delete` to remove the old `store_id` information. If unresolved, **contact R&D**.' # - alert: TiDBMetExpensiveQuery # expr: | # max by (cluster_id, project_id) ( # sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `[expensive_query]` # != `sql="analyze` # != `sql="ANALYZE` # [10m] # ) # ) # ) > 50 # and # max by (cluster_id, project_id) ( # sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `tp add index` # |= `Type:modify column` # [10m] # ) # ) # ) > 0 # for: 1m # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Too much expensive queries" # message: 'TiDB cluster {{ $labels.cluster_id }} just met too much expensive queries' # - alert: TiDBExpensiveQueryCntGt100For1m # expr: | # max by (cluster_id, project_id) ( # sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `[expensive_query]` # != `sql="analyze` # != `sql="ANALYZE` # [10m] # ) # ) # ) > 100 # for: 1m # labels: # severity: major # component: tidb-optimizer # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "A large number of running expensive queries exceeding 60 seconds." # message: 'TiDB cluster {{ $labels.cluster_id }} just met 100+ expensive queries exceeding 1 minute.' # NOTE: kept as-is. Same expr/threshold as the main rule below but for a single # cluster (1379661944646414117) that is excluded from the main rule because its # expensive-query baseline is high. The >500/10m absolute threshold is unreasonable # for this cluster; redesign threshold to be baseline-relative before deletion. - alert: TiDBExpensiveQueryCntGt500AndIncreaseFor10mForAP expr: | (max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123", cluster_id=~"1379661944646414117" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [10m] ) ) ) > 500) and (max by (cluster_id, project_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123", cluster_id=~"1379661944646414117" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [1m] ) ) > 3 * max by (cluster_id, project_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123", cluster_id=~"1379661944646414117" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [1m] offset 10m ) )) for: 0s labels: severity: warning component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance regions: japan annotations: summary: "A large number of running expensive queries exceeding 10 min." message: 'TiDB cluster eagle-insight just met 500+ running expensive queries and is 3x higher than 10min before.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiDBExpensiveQueryCntGt500AndIncreaseFor10m expr: | (max by (cluster_id, project_id) ( sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [10m] ) ) ) > 500) and (max by (cluster_id, project_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [1m] ) ) > 3 * max by (cluster_id, project_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331" , container="tidb"} |= `[expensive_query]` != `sql="analyze` [1m] offset 10m ) )) for: 0s labels: severity: critical response_class: page component: tidb-optimizer oncall_team: tidb_optimizer stability_governance: errorlog-governance regions: japan annotations: summary: "A large number of running expensive queries exceeding 10 min." message: 'TiDB cluster {{ $labels.cluster_id }} just met 500+ running expensive queries and is 3x higher than 10min before.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: PDFailedToPersistStore expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} |= `cluster.go` |= `failed to persist store` [1m] )) ) > 4 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance regions: japan annotations: summary: "PD Failed to Persist Store" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to persist store" errors more than 4 times in the last 1 minute. **This significantly impacts TSO (Timestamp Oracle) and heartbeat mechanisms.** It''s often due to **disk issues**. We recommend checking the **disk status** using Node Exporter.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: PDLoadFromEtcdError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} |= `etcdutil.go` |= `load from etcd meet error` [1m] )) ) > 4 for: 0s labels: severity: major response_class: ticket component: pd oncall_team: pd stability_governance: errorlog-governance regions: japan annotations: summary: "PD Load From Etcd Error" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load from etcd meet error" errors more than 4 times in the last 1 minute. This indicates PD cannot load data stored in Etcd, leading to **scheduler and TSO issues**. Check **disk read/write latency** via Node Exporter. If it''s not a latency problem, escalate to R&D.' # - alert: PDGetTSError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `tso_dispatcher.go` # |= `getTS error` # [5m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "PD Get TSO Error" # message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "getTS error" more than 9 times in the last 5 minutes. This means user TSO requests are failing, causing **SQL execution failures**. It''s usually related to **leader switching or TCP connection disconnections**. These issues often **self-recover**; verify via logs. If not one of these reasons, request R&D support.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: PDFailedToCloseGRPC expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} |= `client.go` |= `failed to close gRPC` [5m] )) ) > 9 for: 0s labels: severity: warning component: pd oncall_team: pd stability_governance: errorlog-governance regions: japan annotations: summary: "PD Failed to Close gRPC Connection" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to close gRPC" errors more than 9 times in the last 5 minutes. This indicates a **connection leak**, which will **consume some CPU resources**. If this persists for 5 minutes, engage R&D for investigation.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: PDSaveToEtcdError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} |= `etcd_kv.go` |= `save to etcd meet error` [1m] )) ) > 9 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance regions: japan annotations: summary: "PD Save to Etcd Error" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "save to etcd meet error" errors more than 9 times in the last 1 minute. This may **affect TSO service and prevent SQL execution**. First, check **disk read/write conditions** via Node Exporter. If normal, escalate to R&D.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: RUTokenStateAnomalyInvalidAssignedToken expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} |= `token_buckets.go` |= `assigned token is invalid` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance regions: japan annotations: summary: "RU Token State Anomaly - Invalid Assigned Token" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "assigned token is invalid"' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: RUTokenStateAnomalyInvalidTokenSlot expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} |= `token_buckets.go` |= `slot token capacity is invalid` [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: pd oncall_team: pd stability_governance: errorlog-governance regions: japan annotations: summary: "RU Token State Anomaly - Invalid Token Slot" message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "slot token capacity is invalid"' # - alert: AbnormalJoinedMember # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `join.go` # |= `there is an abnormal joined member in the current member list` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Abnormal Joined Member" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "there is an abnormal joined member in the current member list" The configuration is incorrect, there is no corresponding setting name' # - alert: PersistMinResolvedTSMeetError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="pd"} # |= `cluster.go` # |= `persist min resolved ts meet error` # [5m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: pd # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Persist Min Resolved TS Meet Error" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "persist min resolved ts meet error" etcd storage failure, usually a storage problem' # - alert: RebuildTableAnalysisJobQueueFail # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `autoanalyze.go` # |= `rebuild table analysis job queue failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Rebuild Table Analysis Job Queue Fail" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "rebuild table analysis job queue failed" The situation of excessive tikv pressure such as tikv unavailable can be ignored, and other clearing alarms will be sent directly. But this is not a serious problem' # - alert: LightningImportFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `backend.go` # |= `import failed` # !~ `context canceled` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Lightning Import Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "import failed" Lightning import failed. Unless the user canceled it, please contact R&D to check the specific reason.' # - alert: DDLCannotFindReorgCtx # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `ddl.go` # |= `cannot find reorgCtx` # [1m] # )) # ) > 0 # for: 3m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "DDL Cannot Find Reorg Ctx" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "cannot find reorgCtx" reorgCtx is used to notify the reorg worker of the status change when the DDL add index or modify column rollback occurs. Continuous occurrence of this error may cause the add index or modify column rollback to fail.' # - alert: FailedToUploadSafepointToPD # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `gc_worker.go` # |= `failed to upload safe point to PD` # [1m] # )) # ) > 0 # for: 2m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Failed To Upload Safepoint To PD" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "failed to upload safe point to PD"' # - alert: LoadDataErrorAndReportCommitWorkPanicked # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `load_data.go` # |= `commitWork panicked` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Load Data Error And Report Commit Work Panicked" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "commitWork panicked" When loading data, inserting data fails because the memory quota is exceeded. You can try temporarily increasing the mem limit or reducing the tidb_dml_batch_size and try again.' # - alert: ImportIntoOrAddIndexReportScanRegionFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `local.go` # |= `scan region failed` # !~ `context canceled` # [1m] # )) # ) > 0 # for: 1m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Import Into Or Add Index Report Scan Region Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "scan region failed"' # - alert: RegionCacheLoadRegionFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `region_cache.go` # |= `load region failure` # [1m] # )) # ) > 0 # for: 2m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Region Cache Load Region Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "load region failure"' # - alert: ValidateReadTSFailedForRequest # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `region_request.go` # |= `validate read ts failed for request` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Validate Read TS Failed For Request" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "validate read ts failed for request"' # - alert: RegionUnavailable # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} # |= `txn.go` # |= `region unavailable` # [1m] # )) # ) > 0 # for: 2m # labels: # severity: major # component: tidb # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "Region Unavailable" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "region unavailable"' # - alert: TiFlashRSSMuchLargerThanLimit # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `DAGDriver.cpp` # |= `[FLASH:Coprocessor:MemoryLimitExceeded] Memory limit (total) exceeded caused by 'RSS(Resident Set Size) much larger than limit'` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiFlash RSS Much Larger Than Limit" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "[FLASH:Coprocessor:MemoryLimitExceeded] Memory limit (total) exceeded caused by "RSS(Resident Set Size) much larger than limit" The query failed due to high TiFlash memory pressure. This could be due to an incorrect query plan, a newly added business query, or other issues causing high TiFlash memory usage. We recommend contacting our oncall team for assistance.' # - alert: TiFlashRSSMuchLargerThanLimit # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `Exception.cpp` # |= `RSS(Resident Set Size) much larger than limit` # [1m] # )) # ) > 4 # for: 0s # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiFlash RSS Much Larger Than Limit" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "RSS(Resident Set Size) much larger than limit" The query failed due to high TiFlash memory pressure. This could be due to an incorrect query plan, a newly added business query, or other issues causing high TiFlash memory usage. We recommend contacting our oncall team for assistance.' # - alert: TiFlashSyncSchemaFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `SchemaSyncService.cpp` # |= `RSS(Resident Set Size) much larger than limit` # [1m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "TiFlash Sync Schema Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "RSS(Resident Set Size) much larger than limit" Broadly speaking, this is an error caused by the inability to connect to the PD.' # - alert: BRBackupCreateStorageFailed # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tikv"} # |= `endpoint.rs` # |= `backup create storage failed` # [1m] # )) # ) > 0 # for: 0s # labels: # severity: major # component: tikv # stability_governance: errorlog-governance # regions: japan # annotations: # summary: "BRBackup Create Storage Failed" # message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} reported "backup create storage failed"' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiDBDDLCancelTimeout expr: | sum by (cluster_id, project_id, tenant_id, instance) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tidb"} |= `State:cancelling` |= `PD server timeout` [1m] ) ) > 0 for: 1m labels: severity: critical response_class: page component: resilience oncall_team: resilience stability_governance: errorlog-tcocp source_tcoc: TCOC-3513 regions: japan annotations: summary: "TiDB DDL Cancel Timeout" message: 'The DDL operation on TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} is in a "cancelling" state and encountering PD server timeouts. This likely indicates slow PD service responses or network issues affecting DDL execution.' # TODO: extract to a generic alert (no tenant filter, applies to all clusters) - alert: TiFlashResourceControlMeetsError expr: | max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( count_over_time( {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089195441286|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646229300|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} |= `LocalAdmissionController.h` |= `LocalAdmissionController.cpp` |= "[ERROR]" != "doRequestGAC got error" != "watchGACLoop failed" [1m] )) ) > 0 for: 0s labels: severity: critical response_class: page component: tiflash oncall_team: tiflash stability_governance: errorlog-governance source_tcoc: TCOC-3352 regions: japan annotations: summary: "TiFlash Resource Control Meets Error" message: 'Tiflash resource control has logical issues, which may cause user queries to be blocked or throttled.' # - alert: TiFlashResourceControlMeetsNetworkError # expr: | # max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) ( # count_over_time( # {cluster_env="prod", tenant_id=~"1372813089201471287|1372813089190911285|1372813089199011282|1379661944646413522|1379661944638454106|1379661944636084123|1372813089191061283|1372813089209283835|1372813089209272332", cluster_id=~"1379661944646415973|10852297103751070537|10602564149339879488|10123293495242577103|1379661944646413522|1379661944638454106|1379661944636084123|1379661944646414117|10947714666900611238|1379661944646416287|1379661944646416286|10244988415975579886|10143988032832670331|10700533136499556764|10766841556523235152" , container="tiflash"} # |= `LocalAdmissionController.h` # |= `LocalAdmissionController.cpp` # |~ "doRequestGAC got error|watchGACLoop failed" # [1m] # )) # ) > 9 # for: 0s # labels: # severity: major # component: tiflash # stability_governance: errorlog-governance # source_tcoc: TCOC-3352 # regions: japan # annotations: # summary: "TiFlash Resource Control Meets Network Error" # message: 'There is a network issue between TiFlash resource control and PD, resulting in failures of RPC for reporting or obtaining RUs.'