apiVersion: monitoring.coreos.com/v1 kind: PrometheusRule metadata: labels: release: self-monitoring name: self-monitoring-dataservice-dataapp-executor namespace: self-monitoring spec: groups: - name: self-monitoring-dataservice-dataapp-executor rules: - alert: DataapiExecutorSessionCreateFailed annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor session create failed. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_session_create_counter{ok="false"}[2m])) by (namespace)) > 0 labels: severity: major component: dataservice - alert: DataapiExecutorSessionNotFound annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor session not found. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_session_not_found{}[2m])) by (namespace)) > 3 or (sum(increase(dataservices_dataapi_executor_session_not_found{}[10m])) by (namespace)) > 10 labels: severity: warning component: dataservice - alert: DataapiExecutorSessionIdEmpty annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor session id empty. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_session_id_empty{}[2m])) by (namespace)) > 0 labels: severity: warning component: dataservice - alert: DataapiExecutorExecSqlRequestInternalError annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor exec sql request 500 failed. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_exec_sql_request_duration_count{ok="false"}[1m])) by (namespace)) > 0 labels: severity: major component: dataservice - alert: DataapiExecutorExecSqlRequestInternalError10min annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor exec sql request 500 failed. Lasts 5min. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_exec_sql_request_duration_count{ok="false"}[1m])) by (namespace)) > 0 for: 10m labels: severity: critical component: dataservice - alert: DataapiExecutorExecSqlRequestInternalError1h annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor exec sql request 500 failed. Lasts 1h. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_exec_sql_request_duration_count{ok="false"}[1m])) by (namespace)) > 0 for: 60m labels: severity: critical component: dataservice - alert: DataapiExecutorDBConnFailed annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor connect database failed. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_database_conn_counter{ok="false"}[2m])) by (namespace)) > 0 labels: severity: major component: dataservice - alert: DataapiExecutorClusterInfoGetFailed annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor get cluster info failed. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_cluster_info_counter{ok="false"}[2m])) by (namespace)) > 0 labels: severity: major component: dataservice - alert: DataappExecutor_PodCpuUsage_GT50 expr: | sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{namespace="dataservice",container="dataservice-dataapp-executor"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-executor",resource="cpu"}) by (pod) > 0.5 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} cpu usage has been greater than 50% for longer than 5 minutes. labels: severity: critical component: dataservice - alert: DataappExecutor_PodMemUsage_GT70 expr: | sum(container_memory_working_set_bytes{namespace="dataservice",container="dataservice-dataapp-executor"}) by (pod) / sum(kube_pod_container_resource_limits{namespace="dataservice",container="dataservice-dataapp-executor",resource="memory"}) by (pod) > 0.7 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, Pod {{ $labels.namespace }} / {{$labels.container}} / {{ $labels.pod }} memory usage has been greater than 70% for longer than 5 minutes. labels: severity: critical component: dataservice - alert: DataappExecutorRegion_QPS_GT50 expr: | sum(increase(dataservices_dataapi_executor_rpc_request_count{}[10m])) / 600 > 50 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-executor region QPS has been greater than 50 for longer than 5 minutes. labels: severity: major component: dataservice - alert: DataappExecutorSessionSLA_LT_999 expr: | sum(increase(dataservices_dataapi_executor_session_cache_counter{ok="true"}[1m]) or vector(1)) by (namespace) / sum(increase(dataservices_dataapi_executor_session_cache_counter{}[1m]) or vector(1)) by (namespace) < 0.999 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-executor session cache SLA < 99.9%. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-ap-northeast-1%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcontainer%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C~%20%60save%20session%20to%20remote%20db%20failed%7Cdelete%20session%20failed%7Cload%20session%20failed%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D) labels: severity: major component: dataservice - alert: DataappExecutorSessionSLA_LT_99 expr: | sum(increase(dataservices_dataapi_executor_session_cache_counter{ok="true"}[1m]) or vector(1)) by (namespace) / sum(increase(dataservices_dataapi_executor_session_cache_counter{}[1m]) or vector(1)) by (namespace) < 0.99 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-executor session cache SLA < 99%. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-ap-northeast-1%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bcontainer%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C~%20%60save%20session%20to%20remote%20db%20failed%7Cdelete%20session%20failed%7Cload%20session%20failed%60%22,%22queryType%22:%22range%22,%22editorMode%22:%22builder%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D) labels: severity: critical component: dataservice - alert: DataappExecutorSessionLatencyP99_GT_1s expr: | histogram_quantile(0.99, sum(rate(dataservices_dataapi_executor_session_cache_duration_bucket{}[1d])) by (le)) > 1000 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-executor session latency P99 great than 1s. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) labels: severity: major component: dataservice - alert: DataappExecutorSessionLatencyP99_GT_3s expr: | histogram_quantile(0.99, sum(rate(dataservices_dataapi_executor_session_cache_duration_bucket{}[1d])) by (le)) > 3000 for: 5m annotations: message: region：{{ $externalLabels.o11y_k8s_id }}, dataservice-dataapp-executor session latency P99 great than 1s. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) labels: severity: critical component: dataservice - alert: DataapiExecutorSessionCreateFailed_Many annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor session create failed count 5 in 2min. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_session_create_counter{ok="false"}[2m])) by (namespace)) > 5 labels: severity: critical component: dataservice - alert: DataapiExecutorExecSqlRequestInternalError_Many annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor exec sql request 500 failed greater than 5 in 1min. [loki link](https://self-monitoring-grafana.observability.tidbcloud.com/explore?orgId=1&left=%7B%22datasource%22:%22Loki-regional-{{ $externalLabels.o11y_k8s_id | reReplaceAll "^o11y-prod-regional-(.+)$" "$1" }}%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22builder%22,%22expr%22:%22%7Bcluster%3D%5C%22o11y-control-plane%5C%22,%20container%3D%5C%22dataservice-dataapp-executor%5C%22%7D%20%7C%3D%20%60ERROR%60%22,%22queryType%22:%22range%22%7D%5D,%22range%22:%7B%22from%22:%22now-30m%22,%22to%22:%22now%22%7D%7D) expr: | (sum(increase(dataservices_dataapi_executor_exec_sql_request_duration_count{ok="false"}[1m])) by (namespace)) > 5 for: 5m labels: severity: critical component: dataservice - alert: DataapiExecutorBuildConnectionInternalError annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor Build Conn Error. expr: | (sum(increase(dataservices_dataapi_executor_build_user_db_conn_duration_count{ok="false"}[1m])) by (namespace)) > 0 labels: severity: major component: dataservice - alert: DataapiExecutorBuildConnectionInternalError_Many annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor Build Conn Error greater than 30 in 2min. expr: | (sum(increase(dataservices_dataapi_executor_build_user_db_conn_duration_count{ok="false"}[2m])) by (namespace)) > 30 for: 5m labels: severity: critical component: dataservice - alert: DataapiExecutorBuildConnectionInternalError_Last10min annotations: message: namespace ：{{ $labels.namespace }} ,region：{{ $externalLabels.o11y_k8s_id }}, Dataapi-region-executor Build Conn Error Last 5min. expr: | (sum(increase(dataservices_dataapi_executor_build_user_db_conn_duration_count{ok="false"}[1m])) by (namespace)) > 0 for: 10m labels: severity: critical component: dataservice