groups:
  - name: serverless-tidb-errorlog-governance
    interval: 30s
    rules:
        - alert: TooManyTikvTxnFileLocks
          expr: |
            sum(count_over_time({cluster_env="prod", namespace="tidb-serverless", container="tikv-log"} |= `preprocess_pending_splits denied` |~ `len:\d{2,}`[30s])) by (cluster) > 0
          for: 5m
          labels:
            severity: critical
            response_class: page
            component: tikv
            oncall_team: tikv_unified
            tier: serverless
            provider_type: aws-free-tier
            stability_governance: serverless-errorlog-governance
          annotations:
            message: 'Kubernetes {{ $labels.cluster }} report to many txn file locks . [Logging](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22P21DC5826856541E4%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22sum%28count_over_time%28%7Bcluster_env%3D%5C%22prod%5C%22,%20namespace%3D%5C%22tidb-serverless%5C%22,%20container%3D%5C%22tikv-log%5C%22%7D%20%7C%3D%20%60preprocess_pending_splits%20denied%60%5B5m%5D%29%29%20by%20%28cluster%29%20%3E%2010%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)'
        - alert: TiFlashMemoryLimitExceeded
          expr: |
            sum(count_over_time({cluster_env="prod", namespace="tidb-serverless", container="errorlog"} |= `error Memory limit (total) exceeded` |~ `len:\d{2,}`[5m])) by (cluster) > 0
          for: 0s
          labels:
            severity: warning
            component: tikv
            oncall_team: tiflash
            tier: serverless
            provider_type: aws-free-tier
            stability_governance: serverless-errorlog-governance
          annotations:
            message: 'Kubernetes {{ $labels.cluster }} report TiFlash memory limit exceeded.'