groups:
  - name: central_worker
    rules:
      - alert: Dataservice_Central_Worker_Privatelink_Controlloop_NotRun
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in Privatelink Controlloop. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-worker%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C%3D%20%60privatelink%20task%20worker%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_worker_task_counter{namespace=~".*-ms",worker_name="privatelink_controlloop"}[2m])) by (namespace, status)) == 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Worker_Privatelink_Controlloop_Failed
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in Privatelink Controlloop. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-worker%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C%3D%20%60privatelink%20task%20worker%60%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_worker_task_counter{namespace=~".*-ms",worker_name="privatelink_controlloop",ok="false"}[5m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Worker_Privatelink_Task_Failed
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in Privatelink Task. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-worker%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C%3D%20%60handler%3Dcommontask_worker%60%20%7C%3D%20%60task_type%3Dprivate_link%60%20%7C~%20%60handler%20completed%7Chandler%20panicked%60%20%21%3D%20%60%5B%5D%60%22,%22queryType%22:%22range%22,%22hide%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_worker_task_counter{namespace=~".*-ms",worker_name="commontask",task_type="private_link",ok="false"}[10m])) by (namespace, status)) > 2
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless