groups:
  - name: serverless-gateway-errorlog-governance
    interval: 30s
    rules:
        - alert: TiDBGatewayInnerError
          expr: |
            sum(count_over_time({cluster_env="prod",namespace="tidb-admin",container="tidb-gateway"} |= "gw_access_log" |= "inner_error" != "backend network break"[1m])) by (cluster) > 500
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb-gateway
            oncall_team: serverless
            tier: serverless
            provider_type: aws-free-tier
            stability_governance: serverless-errorlog-governance
          annotations:
            message: 'Kubernetes cluster {{ $labels.cluster }} reported TiDB Gateway inner error too much. [Logging](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-ap-southeast-1%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_env%3D%5C%22prod%5C%22,%20namespace%3D%5C%22tidb-admin%5C%22,%20container%3D%5C%22tidb-gateway%5C%22%7D%20%7C%3D%20%5C%22gw_access_log%5C%22%20%7C%3D%20%5C%22inner_error%5C%22%20%21%3D%20%5C%22backend%20network%20break%5C%22%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)'
        - alert: TiDBReportPDServerTimeout
          expr: |
            sum(count_over_time({cluster_env="prod", namespace="tidb-admin", container="tidb-gateway"} |= "PD server timeout"[1m])) by (cluster) > 1
          for: 0s
          labels:
            severity: major
            response_class: ticket
            component: tidb
            oncall_team: serverless
            tier: serverless
            provider_type: aws-free-tier
            stability_governance: serverless-errorlog-governance
          annotations:
            message: 'Kubernetes cluster {{ $labels.cluster }} reported pd server timeout error. [Logging](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-east-1%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_env%3D%5C%22prod%5C%22,%20namespace%3D%5C%22tidb-admin%5C%22,%20container%3D%5C%22tidb-gateway%5C%22%7D%20%7C%3D%20%5C%22PD%20server%20timeout%5C%22%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22,%22hide%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)'