groups:
  - name: nextgen-pd-errorlog-governance
    rules:
    - alert: PDLoadFromETCDMeetErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `etcdutil.go`
                      |= `load from etcd meet error`
                    [5m]
                  ))
                ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Loading etcd data is abnormal"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "load from etcd meet error" If the error is not a network error, contact the oncall person directly'

    - alert: PDRevokeLeaseFailedNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `lease.go`
                      |= `revoke lease failed`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Affects tso renewal and pd leader election"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "revoke lease failed" When triggered, you can first check the disk related information. If there is no abnormality, notify the oncall personnel'

    - alert: PDGetTSErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `client.go`
                      |= `[pd] getTS error`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Affect TSO service"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[pd] getTS error" If it is not a network error, you can contact oncall for troubleshooting'

    - alert: PDReportGetTSErrorInTSODispatcherNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `tso_dispatcher.go`
                      |= `[tso] getTS error`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Affect TSO service"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[tso] getTS error" If it is not a network error, you can contact oncall for troubleshooting'

    - alert: PDGetTSErrorAfterProcessingRequestsNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `tso_dispatcher.go`
                      |= `[pd] getTS error`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Affect TSO service"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "[pd] getTS error" If it is not a network error, you can contact oncall for troubleshooting'

    - alert: PDUpdateTSErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `pd.go`
                      |= `updateTS error`
                    [1m]
                  ))
                ) > 0
      for: 2m
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: ""
        message: ''

    - alert: PDFailAddSchedulerWithIndependentConfNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `coordinator.go`
                      |= `can not add scheduler with independent configuration`
                    [1m]
                  ))
                ) > 0
      for: 1m
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Scheduler Cannot Be Initialized"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "can not add scheduler with independent configuration" The storage format may be incompatible. Contact R&D directly'

    - alert: PDRedirectButServerNotLeaderNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `middleware.go`
                      |= `redirect but server is not leader`
                    [5m]
                  ))
                ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Http Request Failed"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "redirect but server is not leader" are mainly ping and health requests, which are generally related to leader switching or TCP connection disconnection. If it is due to these two reasons, you can wait for self-recovery. If it is not due to either, it is recommended to contact R&D. If it is not recovered for more than 5 minutes, contact R&D. http requests have little impact on the business, and the core links are all grpc requests'

    - alert: PDFailPassPrecheckNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `member.go`
                      |= `failed to pass pre-check, check pd leader later`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Failed to expand the pd node"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to pass pre-check, check pd leader later" need to find R&D to troubleshoot'

    - alert: PDFailedToPersistStoreNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `cluster.go`
                      |= `failed to persist store`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "PD Failed to Persist Store"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to persist store" errors more than 4 times in the last 1 minute. **This significantly impacts TSO (Timestamp Oracle) and heartbeat mechanisms.** It''s often due to **disk issues**. We recommend checking the **disk status** using Node Exporter.'

    - alert: PDLoadFromEtcdErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `etcdutil.go`
                      |= `load from etcd meet error`
                    [1m]
                  ))
                ) > 4
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "PD Load From Etcd Error"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "load from etcd meet error" errors more than 4 times in the last 1 minute. This indicates PD cannot load data stored in Etcd, leading to **scheduler and TSO issues**. Check **disk read/write latency** via Node Exporter. If it''s not a latency problem, escalate to R&D.'

    - alert: PDGetTSErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `tso_dispatcher.go`
                      |= `getTS error`
                    [5m]
                  ))
                ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "PD Get TSO Error"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "getTS error" more than 9 times in the last 5 minutes. This means user TSO requests are failing, causing **SQL execution failures**. It''s usually related to **leader switching or TCP connection disconnections**. These issues often **self-recover**; verify via logs. If not one of these reasons, request R&D support.'

    - alert: PDFailedToCloseGRPCNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `client.go`
                      |= `failed to close gRPC`
                    [5m]
                  ))
                ) > 9
      for: 0s
      labels:
        severity: warning
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "PD Failed to Close gRPC Connection"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "failed to close gRPC" errors more than 9 times in the last 5 minutes. This indicates a **connection leak**, which will **consume some CPU resources**. If this persists for 5 minutes, engage R&D for investigation.'

    - alert: PDSaveToEtcdErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `etcd_kv.go`
                      |= `save to etcd meet error`
                    [1m]
                  ))
                ) > 9
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "PD Save to Etcd Error"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "save to etcd meet error" errors more than 9 times in the last 1 minute. This may **affect TSO service and prevent SQL execution**. First, check **disk read/write conditions** via Node Exporter. If normal, escalate to R&D.'

    - alert: RUTokenStateAnomalyInvalidAssignedTokenNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `token_buckets.go`
                      |= `assigned token is invalid`
                    [1m]
                  ))
                ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "RU Token State Anomaly - Invalid Assigned Token"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "assigned token is invalid"'

    - alert: RUTokenStateAnomalyInvalidTokenSlotNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `token_buckets.go`
                      |= `slot token capacity is invalid`
                    [1m]
                  ))
                ) > 0
      for: 0s
      labels:
        severity: critical
        response_class: page
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "RU Token State Anomaly - Invalid Token Slot"
        message: 'PD instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "slot token capacity is invalid"'

    - alert: AbnormalJoinedMemberNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `join.go`
                      |= `there is an abnormal joined member in the current member list`
                    [1m]
                  ))
                ) > 0
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Abnormal Joined Member"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "there is an abnormal joined member in the current member list" The configuration is incorrect, there is no corresponding setting name'

    - alert: PersistMinResolvedTSMeetErrorNextGen
      expr: |
        max by (tidb_cluster_id) (sum by (tidb_cluster_id, instance_id) (
                  count_over_time(
                    {cluster_env="prod", project_id = "", container="pd"}
                      |= `cluster.go`
                      |= `persist min resolved ts meet error`
                    [5m]
                  ))
                ) > 9
      for: 0s
      labels:
        severity: major
        response_class: ticket
        component: pd
        oncall_team: pd
        tier: next-gen
        stability_governance: nextgen-errorlog-governance
      annotations:
        summary: "Persist Min Resolved TS Meet Error"
        message: 'TiDB instance {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} reported "persist min resolved ts meet error" etcd storage failure, usually a storage problem'