apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: self-monitoring-dataservice-dataapp-control
  namespace: self-monitoring
spec:
  groups:
    - name: self-monitoring-dataservice-dataapp-control
      rules:
        - alert: DataapiControlCreatePrivateLinkFailed
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }},  Dataapi-control create private link failed. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)
          expr: |
            (sum(increase(dataservices_dataapi_control_http_requests_total{status != "200", path =~".*create"}[2m]))  by (namespace)) > 0
          labels:
            severity: major
            component: dataservice
        - alert: DataapiControlDeletePrivateLinkFailed
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }},  Dataapi-control delete private link failed. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)
          expr: |
            (sum(increase(dataservices_dataapi_control_http_requests_total{status != "200", path =~".*delete"}[2m]))  by (namespace)) > 0
          labels:
            severity: major
            component: dataservice

        - alert: DataappControl_PodCpuUsage_GT50
          expr: |
            sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="dataservice",container="dataservice-dataapp-control"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-control",resource="cpu"}) by (pod) > 0.5
          for: 5m
          annotations:
            message: Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} cpu usage has been greater than 50% for longer than 5 minutes.
          labels:
            severity: major
            component: dataservice

        - alert: DataappControl_PodMemUsage_GT70
          expr: |
            sum(container_memory_working_set_bytes{namespace="dataservice",container="dataservice-dataapp-control"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-control",resource="memory"}) by (pod) > 0.7
          for: 5m
          annotations:
            message: Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} memory usage has been greater than 70% for longer than 5 minutes.
          labels:
            severity: major
            component: dataservice

        - alert: DataappControlRegion_QPS_GT50
          expr: |
            sum(increase(dataservices_dataapi_control_http_requests_total{}[10m])) / 600 > 50
          for: 5m
          annotations:
            message: dataservice-dataapp-control region HTTP QPS has been greater than 50 for longer than 5 minutes.
          labels:
            severity: major
            component: dataservice