groups:
  - name: central
    rules:
      - alert: Dataservice_Central_Create_APP
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in creat dataapp. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DPOST%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="POST",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Update_APP
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in update dataapp. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DPUT%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%2F:app_id%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="PUT",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps/:app_id"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Delete_APP
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in delete dataapp. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DDELETE%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%2F:app_id%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="DELETE",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps/:app_id"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_List_APP
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in list dataapp. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DGET%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="GET",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Create_ApiKey
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in create apikey. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DPOST%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%2F:app_id%2Fapikeys%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="POST",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps/:app_id/apikeys"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Update_ApiKey
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in update apikey. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DPUT%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%2F:app_id%2Fapikeys%2F:apikey_id%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="PUT",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps/:app_id/apikeys/:apikey_id"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Delete_ApiKey
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, Error in delete apikey. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3DDELETE%60%20%7C%3D%20%60path%3D%2Fapi%2Fv1%2Fdataservices%2Fcentral%2Forgs%2F:org_id%2Fprojects%2F:project_id%2Fapps%2F:app_id%2Fapikeys%2F:apikey_id%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*",method="DELETE",path="/api/v1/dataservices/central/orgs/:org_id/projects/:project_id/apps/:app_id/apikeys/:apikey_id"}[1m])) by (namespace, status)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Recover_ApiKey_Failure
        annotations:
          message: namespace:{{ $labels.namespace }}, Error in recover apikey. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C%3D%20%60DeleteDataApp%20recover%20all%20apikey%20permission%20failed%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_apikey_recover_failed{namespace=~".*-ms"}[1m])) by (namespace)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Http_5xx
        annotations:
          message: namespace:{{ $labels.namespace }}, method:{{ $labels.method }}, path:{{ $labels.path }}, status:{{ $labels.status }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D%5B45%5D%5C%5Cd%2B%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*"}[1m])) by (namespace)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Http_Api_5xx
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*"}[1m])) by (namespace, status, method, path)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Http_Api_5xx_Many
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*"}[1m])) by (namespace)) > 5
        labels:
          severity: critical
          response_class: page
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Http_Api_5xx_5min
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_http_requests_total{namespace=~".*-ms",status=~"5.*"}[1m])) by (namespace)) > 0
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Deploy_Error
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_deploy_duration_count{ok="false"}[2m])) by (namespace)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Deploy_Error_Many
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_deploy_duration_count{ok="false"}[5m])) by (namespace)) > 4
        labels:
          severity: critical
          response_class: page
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Sync_DDB_ERROR
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_sync_ddb{ok="false"}[2m])) by (namespace)) > 0
        labels:
          severity: major
          response_class: ticket
          component: dataservice
          oncall_team: serverless

      - alert: Dataservice_Central_Sync_DDB_ERROR_Many
        annotations:
          message: namespace:{{ $labels.namespace }}, status:{{ $labels.status }}, method:{{ $labels.method }}, path:{{ $labels.path }}. [loki link](https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Loki-aws-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22dataservice-central%5C%22,%20cluster_env%3D%5C%22prod%5C%22%7D%20%7C~%20%60api_code%3D{{ $labels.status }}%60%20%7C%3D%20%60method%3D{{ $labels.method }}%60%20%7C%3D%20%60path%3D{{ $labels.path }}%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D)
        expr: |
          (sum(increase(dataservices_central_sync_ddb{ok="false"}[2m])) by (namespace)) > 4
        labels:
          severity: critical
          response_class: page
          component: dataservice
          oncall_team: serverless