groups:
- name: nextgen-tiflash-errorlog-tcocp
  rules:
    - alert: TiFlashResourceControlMeetsErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `LocalAdmissionController.h`
              |= `LocalAdmissionController.cpp`
              |= "[ERROR]"
              != "doRequestGAC got error"
              != "watchGACLoop failed"
            [1m]
          ))
        ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
        source_tcoc: TCOC-3352
      annotations:
        summary: "TiFlash Resource Control Meets Error"
        message: 'Tiflash resource control has logical issues, which may cause user queries to be blocked or throttled.'
    - alert: TiFlashResourceControlMeetsNetworkErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
          count_over_time(
            {cluster_env="prod", project_id = "", container="tiflash"}
              |= `LocalAdmissionController.h`
              |= `LocalAdmissionController.cpp`
              |~ "doRequestGAC got error|watchGACLoop failed"
            [1m]
          ))
        ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: tiflash
        oncall_team: tiflash
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
        source_tcoc: TCOC-3352
      annotations:
        summary: "TiFlash Resource Control Meets Network Error"
        message: 'There is a network issue between TiFlash resource control and PD, resulting in failures of RPC for reporting or obtaining RUs.'