apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: self-monitoring-dataservice-dataapp-control
  namespace: self-monitoring
spec:
  groups:
    - name: self-monitoring-dataservice-dataapp-control
      rules:
        - alert: DataapiControlCreatePrivateLinkFailed
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }},  Dataapi-control create private link failed. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-control%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)
          expr: |
            (sum(increase(dataservices_dataapi_control_http_requests_total{status != "200", path =~".*create"}[2m]))  by (namespace)) > 0
          labels:
            severity: major
            component: dataservice
        - alert: DataapiControlDeletePrivateLinkFailed
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }},  Dataapi-control delete private link failed. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-control%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D)
          expr: |
            (sum(increase(dataservices_dataapi_control_http_requests_total{status != "200", path =~".*delete"}[2m]))  by (namespace)) > 0
          labels:
            severity: major
            component: dataservice

        - alert: DataappControl_PodCpuUsage_GT50
          expr: |
            sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="dataservice",container="dataservice-dataapp-control"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-control",resource="cpu"}) by (pod) > 0.5
          for: 5m
          annotations:
            message: Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} cpu usage has been greater than 50% for longer than 5 minutes.
          labels:
            severity: critical
            component: dataservice

        - alert: DataappControl_PodMemUsage_GT70
          expr: |
            sum(container_memory_working_set_bytes{namespace="dataservice",container="dataservice-dataapp-control"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-control",resource="memory"}) by (pod) > 0.7
          for: 5m
          annotations:
            message: Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} memory usage has been greater than 70% for longer than 5 minutes.
          labels:
            severity: critical
            component: dataservice

        - alert: DataappControlRegion_QPS_GT50
          expr: |
            sum(increase(dataservices_dataapi_control_http_requests_total{}[10m])) / 600 > 50
          for: 5m
          annotations:
            message: dataservice-dataapp-control region HTTP QPS has been greater than 50 for longer than 5 minutes.
          labels:
            severity: major
            component: dataservice

        - alert: DataappControlRegion_DDBStreamShardConsumber_Offline
          expr: |
            ( time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dataapp.*",worker="ddbstream_shard_consumer"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"endpoints.*",worker="ddbstream_shard_consumer"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"endpoint_publish.*",worker="ddbstream_shard_consumer"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dataapp_clusters.*",worker="ddbstream_shard_consumer"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dev_account_user_api_keys.*",worker="ddbstream_shard_consumer"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dev_account_privilege_role.*",worker="ddbstream_shard_consumer"}) > 120 )
          for: 5m
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-control region ddbstream shard consumber offline. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Prometheus-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_consumer%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22B%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_updater%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22C%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22D%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name!%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22F%22,%22expr%22:%22time()%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D)
          labels:
            severity: critical
            component: dataservice

        - alert: DataappControlRegion_DDBStreamShardUpdater_Offline
          expr: |
            ( time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dataapp.*",worker="ddbstream_shard_updater"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"endpoints.*",worker="ddbstream_shard_updater"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"endpoint_publish.*",worker="ddbstream_shard_updater"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dataapp_clusters.*",worker="ddbstream_shard_updater"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dev_account_user_api_keys.*",worker="ddbstream_shard_updater"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"dev_account_privilege_role.*",worker="ddbstream_shard_updater"}) > 120 )
          for: 5m
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-control region ddbstream shard updater offline. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Prometheus-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_consumer%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22B%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_updater%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22C%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22D%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name!%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22F%22,%22expr%22:%22time()%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D)
          labels:
            severity: critical
            component: dataservice

        - alert: DataappControlRegion_DDBCacheWorker_Offline
          expr: |
            ( time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"ddbcache_worker:dataapp.*",worker="ddbcache_worker"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"ddbcache_worker:endpoints.*",worker="ddbcache_worker"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"ddbcache_worker:endpoint_publish.*",worker="ddbcache_worker"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"ddbcache_worker:dataapp_clusters.*",worker="ddbcache_worker"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"ddbcache_worker:dev_account_user_api_keys.*",worker="ddbcache_worker"}) > 120
              or
              time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name=~"ddbcache_worker:dev_account_privilege_role.*",worker="ddbcache_worker"}) > 120 )
          for: 5m
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-control region ddbcache worker offline. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Prometheus-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_consumer%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22B%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_updater%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22C%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22D%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name!%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22F%22,%22expr%22:%22time()%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D)
          labels:
            severity: critical
            component: dataservice

        - alert: DataappControlRegion_DDBCacheCheckpointsStorage_Offline
          expr: |
            time() - avg(dataservices_dataapi_control_worker_heartbeat_time{name="ddbcache_worker:checkpointstorage",worker="ddbcache_worker"}) > 120
          for: 5m
          annotations:
            message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-control region ddbcache checkpoints storage offline. [loki link](https://self-monitoring-grafana.observability-dev.pingcap.cloud/explore?orgId=1&left=%7B%22datasource%22:%22Prometheus-regional-us-west-2%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_consumer%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22B%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbstream_shard_updater%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22C%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22D%22,%22expr%22:%22dataservices_dataapi_control_worker_heartbeat_time%7Bworker%3D%5C%22ddbcache_worker%5C%22,%20name!%3D%5C%22ddbcache_worker:checkpointstorage%5C%22%7D%22,%22range%22:true,%22editorMode%22:%22code%22%7D,%7B%22refId%22:%22F%22,%22expr%22:%22time()%22%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D)
          labels:
            severity: critical
            component: dataservice