groups:
  - name: serverless-pd-errorlog-governance
    interval: 30s
    rules:
      - alert: PDRuleIsInBadFormat
        expr: |
          sum(count_over_time({cluster_env="prod", container="pd", tanent_id="prod-devtier", namespace="tidb-serverless"} |= `rule is in bad format`[1m])) by (cluster) > 0
        for: 0s
        labels:
          severity: major
          response_class: ticket
          component: pd
          oncall_team: serverless
          tier: serverless
          provider_type: aws-free-tier
          stability_governance: serverless-errorlog-governance
        annotations:
          summary: "PD reported rule is in bad format"
          message: 'Kubernetes cluster {{ $labels.cluster }} reported PD log keyword "rule is in bad format", it may affect PD rule loading. [Logging](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-ap-southeast-1%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_env%3D%5C%22prod%5C%22,%20container%3D%5C%22pd%5C%22,%20tanent_id%3D%5C%22prod-devtier%5C%22%7D%20%7C%3D%20%60rule%20is%20in%20bad%20format%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)'
      - alert: PDLoadRulesFailed
        expr: |
          sum(count_over_time({cluster_env="prod", container="pd", tanent_id="prod-devtier", namespace="tidb-serverless"} |= `load rules failed`[1m])) by (cluster) > 0
        for: 0s
        labels:
          severity: critical
          response_class: page
          component: pd
          oncall_team: serverless
          tier: serverless
          provider_type: aws-free-tier
          stability_governance: serverless-errorlog-governance
        annotations:
          summary: "PD reported load rules failed"
          message: 'Kubernetes cluster {{ $labels.cluster }} reported PD log keyword "load rules failed", it may affect PD rule loading. [Logging](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-ap-southeast-1%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcluster_env%3D%5C%22prod%5C%22,%20container%3D%5C%22pd%5C%22,%20tanent_id%3D%5C%22prod-devtier%5C%22%7D%20%7C%3D%20%60load%20rules%20failed%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)'