groups:
  - name: csng.go-runtime
    rules:
      - alert: GoroutineContinuouslyIncrease
        annotations:
          message: Goroutine of pod {{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} is continuous increasing
          diagnosis: 'Click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Ak6VpZD4z%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22go_goroutines%7Bpod%3D%5C%22{{ $labels.pod }}%5C%22%7D%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D'
        expr: |
          avg(increase(go_goroutines{namespace=~"cluster-ng-service"}[3m])) by (pod)
          > 500
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen

      - alert: GoroutineCountIsTooHigh
        annotations:
          message: Goroutine of pod {{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} is more than 1000
          diagnosis: 'Click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Ak6VpZD4z%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22go_goroutines%7Bpod%3D%5C%22{{ $labels.pod }}%5C%22%7D%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D'
        expr: |
          avg(go_goroutines{namespace=~"cluster-ng-service"}) by (pod)
          > 1000
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen