groups:
- name: swat-errorlog-tcocp
  rules:
    - alert: TiDBDDLCancelTimeout
      expr: |
        sum by (cluster_id, project_id, tenant_id, instance) (
          count_over_time(
            {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", container="tidb"}
              |= `State:cancelling`
              |= `PD server timeout`
            [1m]
          )
        ) > 0
      for: 1m
      labels:
        severity: critical
        response_class: page
        component: tidb-sql
        oncall_team: tidb_sql
        stability_governance: errorlog-tcocp
        source_tcoc: TCOC-3513
      annotations:
        summary: "TiDB DDL Cancel Timeout"
        message: 'The DDL operation on TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} is in a "cancelling" state and encountering PD server timeouts. This likely indicates slow PD service responses or network issues affecting DDL execution.'
    - alert: TiFlashResourceControlMeetsError
      expr: |
        max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) (
          count_over_time(
            {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", container="tiflash"}
              |= `LocalAdmissionController.h`
              |= `LocalAdmissionController.cpp`
              |= "[ERROR]"
              != "doRequestGAC got error"
              != "watchGACLoop failed"
            [1m]
          ))
        ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: tiflash
        oncall_team: tiflash
        stability_governance: errorlog-governance
        source_tcoc: TCOC-3352
      annotations:
        summary: "TiFlash Resource Control Meets Error"
        message: 'Tiflash resource control has logical issues, which may cause user queries to be blocked or throttled.'
    - alert: TiFlashResourceControlMeetsNetworkError
      expr: |
        max by (cluster_id, project_id) (sum by (cluster_id, project_id, instance_id) (
          count_over_time(
            {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", container="tiflash"}
              |= `LocalAdmissionController.h`
              |= `LocalAdmissionController.cpp`
              |~ "doRequestGAC got error|watchGACLoop failed"
            [1m]
          ))
        ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        stability_governance: errorlog-governance
        source_tcoc: TCOC-3352
      annotations:
        summary: "TiFlash Resource Control Meets Network Error"
        message: 'There is a network issue between TiFlash resource control and PD, resulting in failures of RPC for reporting or obtaining RUs.'
#        - alert: TiDBAddIndexDDLJobFinished
#          expr: |
#            sum by (cluster_id, project_id, tenant_id) (
#              count_over_time(
#                {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", container="tidb"}
#                  |= `State:synced`
#                  |= `finish DDL job`
#                  |= `add index`
#                [1m]
#              )
#            ) > 0
#          for: 0s
#          labels:
#            severity: critical
#            component: tidb-sql-data
#            stability_governance: errorlog-tcocp
#            source_tcoc: TCOC-3702
#          annotations:
#            summary: "Add index DDL job finished"
#            message: 'TiDB cluster {{`{{`}} $labels.cluster_id {{`}}`}} just completed an add index DDL job. Please inform the customer to execute "analyze table"'
    - alert: TiDBInnerWorkerPanicked
      expr: |
        sum by (cluster_id, project_id, tenant_id, instance) (
          count_over_time(
            {cluster_env="prod", tenant_id=~"1372813089209061633|1372813089209289635|1372813089209238420|1372813089209256793|.*1372813089209261054", project_id!~"1372813089454536384|1372813089454561820|1372813089454574660|1372813089454576822", container="tidb"}
              |= `innerWorker panicked` |= `invalid memory address or nil pointer dereference`
            [1m]
          )
        ) > 0
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tidb
        oncall_team: resilience
        stability_governance: errorlog-tcocp
        source_tcoc: TCOC-4902
      annotations:
        summary: "TiDB innerWorker panicked"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.cluster_id }} encountered innerWorker panic.'