{ "dashboard_title": "Serverless Tier One Screen", "rows": [ { "title": "Clusters", "panels": [ { "title": "Active Clusters (1d)", "subtitle": null, "expressions": [ "count(sum(increase(gateway_connection_sla_total{cluster_id!=\"\"}[1d])) by (cluster_id) > 0)" ] }, { "title": "Active Clusters (7d)", "subtitle": null, "expressions": [ "count(sum(increase(gateway_connection_sla_total{cluster_id!=\"\"}[7d])) by (cluster_id) > 0)" ] }, { "title": "Active Clusters (30d)", "subtitle": null, "expressions": [ "count(sum(increase(gateway_connection_sla_total{cluster_id!=\"\"}[30d])) by (cluster_id) > 0)" ] }, { "title": "Active Clusters (90d)", "subtitle": null, "expressions": [ "count(sum(increase(gateway_connection_sla_total{cluster_id!=\"\"}[90d])) by (cluster_id) > 0)" ] }, { "title": "TiKV Storage (> 5GiB)", "subtitle": null, "expressions": [ "sum(sum by(keyspace_id) (avg by(keyspace_id, region_id) (tikv_store_size_bytes{namespace=\"tidb-serverless\",type=\"used\"})))", "(\n quantile_over_time(0.5, sum by(keyspace_id) (avg by(keyspace_id, region_id) (tikv_store_size_bytes{namespace=\"tidb-serverless\",type=\"used\"}))[5m]) > 5*1024*1024*1024\n) - on(keyspace_id) group_right() sum by (keyspace_id, cluster_id) (0*gateway_connection_flow_bytes_total)" ] }, { "title": "TiFlash Storage (> 5GiB)", "subtitle": null, "expressions": [ "sum(sum by(keyspace_id) (avg by(keyspace_id, region_id) (tikv_store_size_bytes{namespace=\"tidb-serverless\",type=\"tiflash_used\"})))", "(quantile_over_time(0.5, sum by(keyspace_id) (avg by(keyspace_id, region_id) (tikv_store_size_bytes{namespace=\"tidb-serverless\",type=\"tiflash_used\"}))[5m]) > 5*1024*1024*1024) - on(keyspace_id) group_right() sum by (keyspace_id, cluster_id) (0*gateway_connection_flow_bytes_total)" ] }, { "title": "TiDB RU (> 10000)", "subtitle": null, "expressions": [ "sum(sum(increase(resource_manager_resource_unit_read_request_unit_sum{name!=\"default\",type=~\"|tp\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_write_request_unit_sum{name!=\"default\",type=~\"|tp\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_sql_layer_request_unit_sum{name!=\"default\",type=~\"|tp\"}[1m])) by(name))", "label_replace(\n(sum(increase(resource_manager_resource_unit_read_request_unit_sum{name!=\"default\",type=~\"|tp\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_write_request_unit_sum{name!=\"default\",type=~\"|tp\"}[1m])) by(name) + \nsum(increase(resource_manager_resource_unit_sql_layer_request_unit_sum{name!=\"default\",type=~\"|tp\"}[1m])) by(name)) > 10000, \"keyspace_id\", \"$1\", \"name\", \"(.*)\") - on(keyspace_id) group_right() sum by (keyspace_id, cluster_id) (0*gateway_connection_flow_bytes_total)" ] }, { "title": "Traffic RU (> 10000)", "subtitle": null, "expressions": [ "sum(increase(gateway_connection_flow_bytes_total{namespace=\"tidb-admin\",service!=\"gateway-metrics-inner\",direction=\"out\",cluster_id!=\"\"}[1m])) / 1024", "sum(increase(gateway_connection_flow_bytes_total{namespace=\"tidb-admin\",service!=\"gateway-metrics-inner\",direction=\"out\",cluster_id!=\"\"}[1m])) by (cluster_id) / 1024 > 10000" ] } ] }, { "title": "Storage", "panels": [ { "title": "Current storage size", "subtitle": null, "expressions": [ "sum(pd_cluster_status{type=\"storage_size\"})" ] }, { "title": "Storage capacity", "subtitle": null, "expressions": [ "pd_cluster_status{type=\"storage_capacity\"}" ] }, { "title": "Number of Leader Regions", "subtitle": null, "expressions": [ "sum(pd_cluster_status{type=\"leader_count\"})" ] }, { "title": "Store Region size", "subtitle": null, "expressions": [ "pd_scheduler_store_status{type=\"store_used\"}" ] }, { "title": "Total Region Size Increase Per Day", "subtitle": null, "expressions": [ "sum(delta(pd_scheduler_store_status{type=\"store_used\"}[1d]))" ] }, { "title": "Store Region count", "subtitle": null, "expressions": [ "pd_scheduler_store_status{type=\"region_count\"}" ] }, { "title": "Total Region Count Increase Per Day", "subtitle": null, "expressions": [ "sum(delta(pd_scheduler_store_status{type=\"region_count\"}[1m]))" ] } ] }, { "title": "TiDB", "panels": [ { "title": "Active Cluster Num", "subtitle": null, "expressions": [ "count(sum(tidb_server_connections{}[$__range]) by (cluster_id))" ] }, { "title": "TiDB Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{ namespace=\"tidb-pool\", pod=~\"tidb-server.*\"} > 0)", "count by (phase) (kube_pod_status_phase{ namespace=\"tidb-pool\", pod=~\".*playground.*\"}[10m] > 0)" ] }, { "title": "Query OPS", "subtitle": null, "expressions": [ "sum(rate(tidb_server_query_total[1m])) by (result)", "sum(rate(tidb_server_query_total{result=\"OK\"}[1m] offset 1d))" ] }, { "title": "VIP TiDB Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{ namespace=~\"vip-tidb-pool\", pod=~\"tidb-server.*\"} > 0)", "count by (phase) (kube_pod_status_phase{ namespace=\"tidb-pool\", pod=~\".*playground.*\"}[10m] > 0)" ] }, { "title": "Cluster QPS VIP", "subtitle": null, "expressions": [ "sum(rate(tidb_server_query_total{namespace=~\".*vip-tidb-pool\"}[1m])) by (cluster_id) > 10" ] }, { "title": "VIP TiDB Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{ namespace=~\"vip-tidb-pool\", pod=~\"tidb-server.*\"} > 0)", "count by (phase) (kube_pod_status_phase{ namespace=\"tidb-pool\", pod=~\".*playground.*\"}[10m] > 0)" ] }, { "title": "Max Go GC Duration", "subtitle": null, "expressions": [ "max(increase(go_gc_cycles_total_gc_cycles_total{container=\"tidb\"}[1m])) by (namespace) * max(go_gc_duration_seconds{container=\"tidb\"}) by (namespace)" ] }, { "title": "SVIP TiDB Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{ namespace=~\".*super-vip-tidb-pool\", pod=~\"tidb-server.*\"} > 0)", "count by (phase) (kube_pod_status_phase{ namespace=\"tidb-pool\", pod=~\".*playground.*\"}[10m] > 0)" ] }, { "title": "Statements QPS", "subtitle": null, "expressions": [ "sum(rate(tidb_executor_statement_total[1m])) by (type)" ] }, { "title": "Duration VIP", "subtitle": null, "expressions": [ "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{namespace=~\".*vip-tidb-pool\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{namespace=~\".*vip-tidb-pool\"}[1m])) by (le))", "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{namespace=~\".*vip-tidb-pool\"}[1m])) by (le))", "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{namespace=~\".*vip-tidb-pool\"}[1m])) by (le))" ] } ] }, { "title": "TiKV", "panels": [ { "title": "TiKV CPU usage", "subtitle": null, "expressions": [ "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", container=\"tikv\"}[1m])) by (instance)" ] }, { "title": "TiKV Memory usage", "subtitle": null, "expressions": [ "avg(process_resident_memory_bytes{ namespace=\"tidb-serverless\", container=\"tikv\"}) by (instance)" ] }, { "title": "Raft Store CPU Usage", "subtitle": null, "expressions": [ "sum(rate(tikv_thread_cpu_seconds_total{ namespace=\"tidb-serverless\", name=~\"(rfstore|raftstore).*\", container=\"tikv\"}[1m])) by (instance)" ] }, { "title": "Engine memory usage", "subtitle": null, "expressions": [ "sum(kv_engine_mem_size_bytes) by (db, type)" ] }, { "title": "Raft Engine WAL duration", "subtitle": null, "expressions": [ "sum(delta(tikv_raftstore_store_write_raftdb_duration_seconds_bucket[1m])) by (le)" ] }, { "title": "gRPC KvCommit duration", "subtitle": null, "expressions": [ "sum(delta(tikv_grpc_msg_duration_seconds_bucket{type=\"kv_commit\"}[1m])) by (le)" ] }, { "title": "S3 Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(kv_engine_dfs_latency_seconds_bucket{type = \"get\"}[1m])) by (le))", "histogram_quantile(0.99, sum(rate(kv_engine_dfs_latency_seconds_bucket{type = \"put\"}[1m])) by (le))" ] }, { "title": "Throttle Limiter", "subtitle": null, "expressions": [ "sum(rate(kv_engine_limiter_throttle_counter[1m])) by (limiter, instance) > 0" ] }, { "title": "S3 Throughput", "subtitle": null, "expressions": [ "sum(rate(kv_engine_dfs_throughput_bytes[1m])) by (type)", "sum(rate(kv_engine_dfs_throughput_bytes[1m])) by (type,instance)", "sum(increase(kv_engine_dfs_throughput_bytes[1m])/60*9) by (type)" ] }, { "title": "Thread CPU Usage", "subtitle": null, "expressions": [ "max(rate(tikv_thread_cpu_seconds_total{ namespace=\"tidb-serverless\", container=\"tikv\"}[2m])) by (name)" ] }, { "title": "S3 OPM", "subtitle": null, "expressions": [ "sum(increase(kv_engine_dfs_latency_ms_count[1m])) by (type)" ] }, { "title": "TiKV Storage Usage Rate", "subtitle": null, "expressions": [ "(1-((sum(tikv_store_size_bytes{ namespace=\"tidb-serverless\", type=\"available\"}) by (instance)) / (sum(tikv_store_size_bytes{ namespace=\"tidb-serverless\", type=\"capacity\", container=\"tikv\"}) by (instance))))" ] }, { "title": "Status API QPS", "subtitle": null, "expressions": [ "sum(rate(tikv_status_request_duration_seconds_count[1m])) by (req)" ] }, { "title": "Level Write Throughput", "subtitle": null, "expressions": [ "sum(rate(kv_engine_level_write_bytes)) by (level)" ] }, { "title": "Engine Cache Miss", "subtitle": null, "expressions": [ "sum(rate(kv_engine_cache_miss[30s])) by (instance)" ] }, { "title": "Status API Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_status_request_duration_seconds_bucket{namespace=\"tidb-serverless\"}[$__rate_interval])) by (le, req))" ] }, { "title": "KV Backoff OPS", "subtitle": null, "expressions": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)" ] }, { "title": "Lock resolve times", "subtitle": null, "expressions": [ "sum(rate(tidb_tikvclient_lock_resolver_actions_total[1m])) by (type)" ] }, { "title": "KV Backoff OPS", "subtitle": null, "expressions": [ "sum(rate(tidb_tikvclient_region_err_total[1m])) by (type)", "sum(rate(tidb_tikvclient_region_err_total{EXTERNAL_LABELtype=\"server_is_busy\"}[1m]))" ] }, { "title": "IA size / capacity", "subtitle": null, "expressions": [ "sum(tikv_engine_size_bytes {db=\"kv\", type=\"ia_kv\"})", "sum(tikv_engine_size_bytes{db=\"kv\", type=\"ia\"})", "sum(tikv_engine_size_bytes{db=\"kv\", type=\"ia\"}) by (instance)", "sum(kv_engine_ia_main_queue_capacity{component=\"tikv\"}) by (instance)", "sum(kv_engine_ia_small_queue_capacity{component=\"tikv\"}) by (instance)", "sum(tikv_engine_size_bytes {db=\"kv\", type=\"ia_kv\"}) by (instance)" ] }, { "title": "IA read segments count", "subtitle": null, "expressions": [ "sum(rate(kv_engine_ia_read_segment_duration_seconds_count[1m])) by (container)", "sum(rate(kv_engine_ia_read_segment_cache_miss[1m])) by (container)", "sum(rate(tikv_worker_remote_cop_prefetch_duration_seconds_count[1m])) by (container)" ] }, { "title": "IA read segments duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(kv_engine_ia_read_segment_duration_seconds_bucket[1m])) by (le,container))" ] }, { "title": "IA read segments cache miss rate", "subtitle": null, "expressions": [ "sum(rate(kv_engine_ia_read_segment_cache_miss[1m])) by (container) / sum(rate(kv_engine_ia_read_segment_duration_seconds_count[1m])) by (container)", "sum(rate(kv_engine_ia_read_segment_cache_miss{instance=~\".*canary.*\"}[1m])) by (instance) / sum(rate(kv_engine_ia_read_segment_duration_seconds_count{instance=~\".*canary.*\"}[1m])) by (instance)" ] }, { "title": "Storage class", "subtitle": null, "expressions": [ "sum(rate(kv_engine_sc_transition_counter[1m])) by (type)" ] }, { "title": "Raft log lag", "subtitle": null, "expressions": [ " histogram_quantile(1, sum(rate(tikv_raftstore_log_lag_bucket[1m])) by (le, instance, region))" ] }, { "title": "region_id count by instance", "subtitle": null, "expressions": [ "sum(count(tikv_store_size_bytes{type=\"used\"})by(instance,region_id)) by (instance)" ] } ] }, { "title": "TiKV Worker", "panels": [ { "title": "CPU usage", "subtitle": null, "expressions": [ "sum(rate(container_cpu_usage_seconds_total{namespace=\"tidb-serverless\", container=\"cop-worker\"}[1m]))", "sum(kube_pod_container_resource_limits{container=\"cop-worker\",namespace=\"tidb-serverless\",resource=\"cpu\"})", "sum(rate(container_cpu_usage_seconds_total{namespace=\"tidb-serverless\", container=\"tikv-worker\"}[1m]))", "sum(kube_pod_container_resource_limits{container=\"tikv-worker\",namespace=\"tidb-serverless\",resource=\"cpu\"})" ] }, { "title": "Memory usage (Cop Worker)", "subtitle": null, "expressions": [ "sum(process_resident_memory_bytes{container=\"cop-worker\"}) by (pod)", "min(container_spec_memory_limit_bytes{container=\"cop-worker\"})", "sum(kv_engine_ia_manager_segments_memory_size{container=\"cop-worker\"}) by (pod)" ] }, { "title": "Memory usage (TiKV Worker)", "subtitle": null, "expressions": [ "sum(process_resident_memory_bytes{container=\"tikv-worker\"}) by (pod)", "min(container_spec_memory_limit_bytes{container=\"tikv-worker\"})", "sum(tikv_worker_memory_limiter_current_used{container=\"tikv-worker\"}) by (pod)", "sum(rate(kv_engine_remote_compact_exceed_memory_limit_counter[30s]))" ] }, { "title": "Native BR", "subtitle": null, "expressions": [ "sum(increase(native_br_backup_success[1m]))", "sum(increase(native_br_backup_error[1m]))", "sum(increase(tikv_worker_native_br_counter[1m])) by (type)" ] }, { "title": "Native BR Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_worker_native_br_duration_seconds_bucket[5m])) by (le,type))", "sum(rate(tikv_worker_native_br_duration_seconds_sum[5m])) by (type) / sum(rate(tikv_worker_native_br_duration_seconds_count[5m])) by (type)" ] }, { "title": "Restore Keyspace Duration", "subtitle": null, "expressions": [ "sum(delta(tikv_worker_native_br_duration_seconds_bucket{type=\"restore_keyspace\"}[1m])) by (le)" ] }, { "title": "TiKV/Cop Worker Duration (P99)", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_worker_remote_cop_snapshot_duration_seconds_bucket[5m])) by (le,container))", "histogram_quantile(0.99, sum(rate(tikv_worker_remote_cop_prefetch_duration_seconds_bucket[5m])) by (le,container))", "histogram_quantile(0.99, sum(rate(tikv_worker_remote_cop_request_duration_seconds_bucket[5m])) by (le,container))" ] }, { "title": "Cop Worker IA Prefetch Cache Hit", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tikv_worker_remote_cop_prefetch_cache_hit_percent_bucket{container=\"cop-worker\"}[1m])) by (le))" ] }, { "title": "Cop Worker QPS", "subtitle": null, "expressions": [ "sum(rate(tikv_worker_remote_cop_request_duration_seconds_count{container=\"cop-worker\"}[1m]))" ] }, { "title": "TiKV Worker QPS", "subtitle": null, "expressions": [ "sum(rate(tikv_worker_remote_cop_request_duration_seconds_count{container=\"tikv-worker\"}[1m]))" ] }, { "title": "Restore keyspace bytes", "subtitle": null, "expressions": [ "native_br_restore_pending_data_size", "rate(native_br_restored_data_size)", "rate(native_br_restored_kv_size)", "native_br_restored_data_size", "native_br_restored_kv_size" ] } ] }, { "title": "TiKV masked", "panels": [ { "title": "TiKV CPU usage (masked)", "subtitle": null, "expressions": [ "label_replace(sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", container=\"tikv\"}[1m])) by (instance), \"instance_rename\", \"$2\", \"instance\", \"(tikv-.*)-((p[0-9]-)?tikv-[0-9]*)\")" ] }, { "title": "Raft Store CPU Usage (masked)", "subtitle": null, "expressions": [ "label_replace(sum(rate(tikv_thread_cpu_seconds_total{ namespace=\"tidb-serverless\", name=~\"raftstore_.*\", container=\"tikv\"}[1m])) by (instance), \"instance_rename\", \"$2\", \"instance\", \"(tikv-.*)-((p[0-9]-)?tikv-[0-9]*)\")" ] }, { "title": "TiKV Memory usage(masked)", "subtitle": null, "expressions": [ "label_replace(avg(process_resident_memory_bytes{ namespace=\"tidb-serverless\", container=\"tikv\"}) by (instance), \"instance_rename\", \"$2\", \"instance\", \"(tikv-.*)-((p[0-9]-)?tikv-[0-9]*)\")" ] } ] }, { "title": "PD", "panels": [ { "title": "PD CPU usage", "subtitle": null, "expressions": [ "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster(|-standby|-witness)-pd.*\"}[1m])) by (instance)", "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"pd-tso-server.*\"}[1m])) by (instance)", "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"pd-sche-server.*\"}[1m])) by (instance)", "sum(rate(process_cpu_seconds_total{ namespace=\"tidb-serverless\", pod=~\"pd-resource-manager-server.*\"}[1m])) by (instance)" ] }, { "title": "PD Memory usage", "subtitle": null, "expressions": [ "avg(process_resident_memory_bytes{ namespace=\"tidb-serverless\", pod=~\".*pd.*\"}) by (instance)" ] }, { "title": "PD GC safepoint", "subtitle": null, "expressions": [ "pd_gc_gc_safepoint{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-pd.*\"}" ] }, { "title": "Pd Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-pd.*\"} > 0)", "increase(kube_pod_container_status_restarts_total{ namespace=\"tidb-serverless\", container=\"pd\"}[1m])" ] }, { "title": "PD ETCD Storage", "subtitle": null, "expressions": [ "etcd_debugging_mvcc_db_total_size_in_bytes{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-pd.*\"}" ] }, { "title": "PD ETCD Disk Operations", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-pd.*\"}[5m])) by (pod, le))", "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{ namespace=\"tidb-serverless\", pod=~\"serverless-cluster-pd.*\"}[5m])) by (pod, le))" ] }, { "title": "PD ETCD Transaction Rate", "subtitle": "The rate of PD handling etcd transactions", "expressions": [ "sum(rate(pd_txn_handle_txns_duration_seconds_count{}[5m])) by (instance, result)" ] }, { "title": "PD ETCD Transaction Duration", "subtitle": "The time consumed of handling etcd transactions", "expressions": [ "histogram_quantile(0.99, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{}[5m])) by (instance, result, le))", "histogram_quantile(0.90, sum(rate(pd_txn_handle_txns_duration_seconds_bucket{}[5m])) by (instance, result, le))" ] }, { "title": "TSO Handle Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket{}[$__rate_interval])) by (le, group))" ] }, { "title": "TSO Operation Duration", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(pd_cluster_tso_operation_duration_seconds_bucket{}[$__rate_interval])) by (le, group, dc, type))" ] }, { "title": "TSO Events", "subtitle": null, "expressions": [ "sum(increase(pd_tso_events[$__rate_interval])) by (group, dc, type)" ] }, { "title": "Keyspace Group State", "subtitle": null, "expressions": [ "tso_keyspace_group_state{}[$__rate_interval]" ] } ] }, { "title": "TiFlash", "panels": [ { "title": "CPU", "subtitle": "The CPU usage of each TiKV instance", "expressions": [ "rate(tiflash_proxy_process_cpu_seconds_total{}[1m])" ] }, { "title": "Memory", "subtitle": "The memory usage per TiKV instance", "expressions": [ "avg(tiflash_proxy_process_resident_memory_bytes{}) by (instance)" ] }, { "title": "TiFlash Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{namespace=\"tidb-serverless\", pod=~\"tiflash-.*\"} > 0)" ] } ] }, { "title": "Gateway", "panels": [ { "title": "SLA", "subtitle": null, "expressions": [ "1 - sum(increase(gateway_connection_sla_total{cluster_id!=\"\",status=~\"inner_error|tidb_error|cancel\"}[1m])) /\nsum(increase(gateway_connection_sla_total{cluster_id!=\"\",status!=\"missmatch\"}[1m]))", "sum(increase(gateway_connection_sla_total{cluster_id!=\"\",status=~\"normal|exter_warn|inner_warn|design_error\"}[1m])) by (tidb_pool,job) / \nsum(increase(gateway_connection_sla_total{cluster_id!=\"\",status!=\"missmatch\"}[1m])) by (tidb_pool,job) " ] }, { "title": "Fetch Manager Failure Count", "subtitle": null, "expressions": [ "sum(increase(gateway_get_backendlist_total{stage=\"fetch_manager_failed\"}[10m])) by (cluster_id)" ] }, { "title": "CPU Request Usage", "subtitle": null, "expressions": [ "rate(container_cpu_usage_seconds_total{namespace=\"tidb-admin\", pod=~\"tidb-gateway-.*|privatelink.*|public.*\",image=\"\"}[$__rate_interval]) / on(pod) kube_pod_container_resource_limits{resource=\"cpu\"}" ] }, { "title": "MEM Request Usage", "subtitle": null, "expressions": [ "sum(container_memory_working_set_bytes{ namespace=\"tidb-admin\",pod=~\"tidb-gateway-.*|privatelink.*|public.*\",container!=\"POD\", container!=\"\", image!=\"\"})by(pod)/\nsum(kube_pod_container_resource_requests{namespace=\"tidb-admin\",pod=~\"tidb-gateway-.*|privatelink.*|public.*\"}) by (pod)" ] }, { "title": "Connection Events every 10 minutes", "subtitle": null, "expressions": [ "sum(increase(gateway_connection_total[10m])) by (product, stage)" ] }, { "title": "Infra request count (GW P95 for 1min)", "subtitle": null, "expressions": [ "sum(increase(infra_client_get_duration_seconds_count{}[1m])) by (name)", "sum(increase(infra_client_cache_get_total{from=\"lru\"}[1m])) by (name)", "sum(increase(infra_client_get_index_duration_seconds_count{job=~\"gateway-.*\"}[1m])) by (name)", "sum(increase(infra_client_cache_get_index_total{job=~\"gateway-.*\"}[1m])) by (name)" ] }, { "title": "Infra request latency (GW P95 for 1min)", "subtitle": null, "expressions": [ "histogram_quantile(0.95, sum(rate(infra_client_get_duration_seconds_bucket{}[1m])) by (le,name))", "histogram_quantile(0.95, sum(rate(infra_client_get_index_duration_seconds_bucket{}[1m])) by (le,name))" ] }, { "title": "Infra Redis Reset (GW for 1min)", "subtitle": null, "expressions": [ "increase(infra_consumer_redis_on_connect_total{}[1m])" ] }, { "title": "Infra Redis Message(GW for 1min)", "subtitle": null, "expressions": [ "increase(infra_consumer_redis_message_total{}[1m])" ] }, { "title": "Bandwidth (bps)", "subtitle": null, "expressions": [ "sum by(service, direction) (irate(gateway_connection_flow_bytes_total[1m]))" ] }, { "title": "Connection per Stage P95 duration (ms)", "subtitle": null, "expressions": [ "1000 * histogram_quantile(0.95, sum(rate(gateway_connection_seconds_bucket{stage!=\"session_total\"}[1m])) by (stage, le))" ] }, { "title": "Connections Error Type Rate[12h]", "subtitle": null, "expressions": [ "sum(increase(gateway_connection_error_type_total[12h])) by (error_type) / on(service) group_left sum(increase(gateway_connection_error_type_total[12h]))", "sum(increase(gateway_connection_error_type_total[12h])) by (service, error_type) / on(service) group_left sum(increase(gateway_connection_error_type_total[12h])) by (service)" ] }, { "title": "clients driver total", "subtitle": null, "expressions": [ "sum (gateway_clients_driver_total) by (client_name)" ] } ] }, { "title": "Manager", "panels": [ { "title": "Manager Cache Size", "subtitle": null, "expressions": [ "tidbmanager_cache_size" ] }, { "title": "Manager Free Cache Size", "subtitle": null, "expressions": [ "tidbmanager_cache_size{type=\"free\", job=\"manager\"}" ] }, { "title": "Assign Tidb Fail Count", "subtitle": null, "expressions": [ "sum(increase(tidbmanager_assign_tidb_seconds_count{stage=\"total_fail\"}[1m])) by (token) > 0" ] }, { "title": "Activate Tidb(P99/5m)", "subtitle": null, "expressions": [ "histogram_quantile(0.99, sum(rate(tidbmanager_activate_seconds_bucket[5m])) by (le, rlt))" ] }, { "title": "Infra apiserver request count (5m)", "subtitle": null, "expressions": [ "(1 - (sum(increase(gateway_connection_sla_total{tidb_pool!~\".*vip.*\",cluster_id!=\"\",status=~\"inner_error|tidb_error|cancel\", cluster!~\"freetier/eks/prod-us-east-1-f0[23]\"}[10m])) by (tidb_pool, pod, cluster_id) /\n sum(increase(gateway_connection_sla_total{tidb_pool!~\".*vip.*\",cluster_id!=\"\",status!=\"missmatch\", cluster!~\"freetier/eks/prod-us-east-1-f0[23]\"}[10m])) by (tidb_pool, pod, cluster_id)))" ] }, { "title": "EKS apiserver request count (5m)", "subtitle": null, "expressions": [ "sum(increase(rest_client_requests_total{namespace=\"tidb-admin\", job=\"manager\", host=\"172.20.0.1:443\"}[5m])) by (method, code)" ] }, { "title": "Infra apiserver request latency (P95 for 24h)", "subtitle": null, "expressions": [ "histogram_quantile(0.95, sum(rate(rest_client_request_latency_seconds_bucket{namespace=\"tidb-admin\", job=\"manager\", url=~\".*?aws.tidbcloud.com:443/apis/.*/%7Bname%7D.*\"}[24h])) by (le,url,verb))" ] }, { "title": "Manager Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{namespace=\"tidb-admin\",pod=~\"manager.*\"} > 0)", "increase(kube_pod_container_status_restarts_total{ namespace=\"tidb-admin\",container=\"manager\"}[1m])", "sum(label_replace(increase(kube_pod_container_status_restarts_total{cluster=~\"freetier.*\",job=\"manager\"}[5m])>=1 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{cluster=~\"freetier.*\",reason='OOMKilled',job=\"manager\"}==1,\"cluster_id\",\"$1\",\"pod\",\"db(.*)-(.*)-(.*)\"))by(cluster_id)" ] }, { "title": "Local apiserver request latency (P95 for 24h)", "subtitle": null, "expressions": [ "histogram_quantile(0.95, sum(rate(rest_client_request_latency_seconds_bucket{namespace=\"tidb-admin\", job=\"manager\", url=~\".*?172.\\\\d+.\\\\d+.\\\\d+:443/.*/%7Bname%7D.*\"}[24h])) by (le,url,verb))" ] }, { "title": "Infra apiserver request count (for 24h)", "subtitle": null, "expressions": [ "sum(increase(rest_client_request_latency_seconds_count{namespace=\"tidb-admin\", job=\"manager\", url=~\".*?aws.tidbcloud.com:443/apis/.*/%7Bname%7D.*\"}[24h])) by (url,verb)" ] }, { "title": "Manager Scaling Metric (CPU) > 50%", "subtitle": null, "expressions": [ "(rate(container_cpu_usage_seconds_total{namespace=\"vip-tidb-pool\",container=\"tidb\"}[$__rate_interval]) / on(pod) (container_spec_cpu_quota{namespace=\"vip-tidb-pool\",container=\"tidb\"}/100000)) > 0.5" ] }, { "title": "Local apiserver request count (for 24h)", "subtitle": null, "expressions": [ "sum(increase(rest_client_request_latency_seconds_count{namespace=\"tidb-admin\", job=\"manager\", url=~\".*?172.\\\\d+.\\\\d+.\\\\d+:443/.*/%7Bname%7D.*\"}[24h])) by (url,verb)" ] }, { "title": "Cluster Creation(Running)/1m", "subtitle": null, "expressions": [ "sum(increase(cluster_creation_seconds_count{status=\"Running\"}[1m])) by (cluster_type)>0", "sum(increase(cluster_creation_seconds_sum{status=\"Running\"}[1m])) by (cluster_type) / sum(increase(cluster_creation_seconds_count{status=\"Running\"}[1m])) by (cluster_type)", "histogram_quantile(0.95, sum(cluster_creation_seconds_bucket{status=\"Running\"}) by (le, cluster_type, is_restore))" ] }, { "title": "Manager Scaling Metric (MEM)", "subtitle": null, "expressions": [ "container_memory_working_set_bytes{namespace=\"vip-tidb-pool\",container=\"tidb\"} / on(pod) container_spec_memory_limit_bytes{namespace=\"vip-tidb-pool\",container=\"tidb\"}" ] }, { "title": "Pool Node CPU Rate(By Labels)", "subtitle": null, "expressions": [ "1-:node_cpu_idle:kube_node_labels{label_serverless_tidbcloud_com_node=\"tidb\"}" ] }, { "title": "Pool Namespace CPU (By Container)", "subtitle": null, "expressions": [ "sum(rate(container_cpu_usage_seconds_total{namespace=\"vip-tidb-pool\",image=\"\"}[$__rate_interval])) by (node)" ] }, { "title": "Tidb AutoScaling", "subtitle": null, "expressions": [ "sum(increase(tidbautoscaling_scale_total[1m])) by (direction, pool_ns)" ] }, { "title": "Evicter", "subtitle": null, "expressions": [ "tidbpodevicter_evict_total{}", "tidbpodevicter_force_evict_total{}" ] }, { "title": "Prevent CA Scale Up", "subtitle": null, "expressions": [ "noderebalance_prevent_ca_scale_up" ] }, { "title": "NodeRebalance Evict Node", "subtitle": null, "expressions": [ "increase(noderebalance_evict_node_total)>0" ] }, { "title": "Pool Abnormal Pods", "subtitle": null, "expressions": [ "sum(tidbscaler_abnormal_tidb) by (pool_ns)" ] }, { "title": "Keyspace pool capacity", "subtitle": null, "expressions": [ "avg(keyspace_available{namespace=\"tidb-admin\", job=\"manager\"}) by (pool_ns)" ] }, { "title": "Pool Old Revision Pod Count", "subtitle": null, "expressions": [ "sum(tidbupgrader_old_pod_assign_cache_size+tidbupgrader_old_pod_free_cache_size) by (pool_ns)" ] }, { "title": "Upgrader Error Duration", "subtitle": null, "expressions": [ "sum(tidbupgrader_upgrader_error_duration) by (pool_ns)" ] }, { "title": "Manager CPU", "subtitle": null, "expressions": [ "rate(container_cpu_usage_seconds_total{namespace=\"tidb-admin\",pod=~\"manager-.*\",image=\"\"}[$__rate_interval]) / on(pod) kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"tidb-admin\",pod=~\"manager-.*\"}" ] }, { "title": "Manager MEMORY", "subtitle": null, "expressions": [ "container_memory_working_set_bytes{pod=~\"manager-.*\",container!=\"POD\",container!=\"\",image!=\"\",namespace=\"tidb-admin\"}/on(pod)\nkube_pod_container_resource_limits{resource=\"memory\",namespace=\"tidb-admin\",pod=~\"manager-.*\"}" ] }, { "title": "Read O11y Metrics Failed", "subtitle": null, "expressions": [ "sum(increase(utilsquery_prom_query_seconds_count{reason!=\"ok\"}[1m])) by (reason)" ] }, { "title": "Upgrader Delete Pod Duration", "subtitle": null, "expressions": [ "sum(tidbupgrader_pod_deletion_duration) by (pod_name)" ] } ] }, { "title": "ETCD", "panels": [ { "title": "Instance Status", "subtitle": "1 up\n0 down", "expressions": [ "up{ job=\"etcd-svc\"}" ] }, { "title": "Instance Has Leader", "subtitle": null, "expressions": [ " etcd_server_has_leader{job=\"etcd-svc\", namespace=\"tidb-admin\"}" ] }, { "title": "ETCD Leader Change Count", "subtitle": null, "expressions": [ "increase((max by (job) (etcd_server_leader_changes_seen_total{job=\"etcd-svc\", namespace=\"tidb-admin\"}) or 0*absent(etcd_server_leader_changes_seen_total{job=\"etcd-svc\",namespace=\"tidb-admin\"}))[15m:1m]) >= 3", "sum(delta(etcd_server_leader_changes_seen_total{namespace=\"tidb-admin\", container=\"etcd\"}[1d]))" ] }, { "title": "ETCD Keys Count", "subtitle": null, "expressions": [ "etcd_debugging_mvcc_keys_total{container=\"etcd\", namespace=\"tidb-admin\"}", "sum(delta(etcd_server_leader_changes_seen_total{namespace=\"tidb-admin\", container=\"etcd\"}[1d]))" ] }, { "title": "compaction keys last 5min", "subtitle": null, "expressions": [ "rate(etcd_debugging_mvcc_db_compaction_keys_total{container=\"etcd\",namespace=\"tidb-admin\"}[5m])" ] } ] }, { "title": "Cluster Proxy", "panels": [ { "title": "Replica Num", "subtitle": null, "expressions": [ "kube_deployment_status_replicas_available{deployment=\"dbaas-proxy\", cluster=~\"freetier.*\"}" ] } ] }, { "title": "Cluster Autoscaler", "panels": [ { "title": "Unschedulable Pods", "subtitle": null, "expressions": [ "cluster_autoscaler_unschedulable_pods_count" ] }, { "title": "Scaled Down Nodes", "subtitle": null, "expressions": [ "cluster_autoscaler_scaled_down_nodes_total" ] }, { "title": "Scaled Up Nodes", "subtitle": null, "expressions": [ "cluster_autoscaler_scaled_up_nodes_total" ] } ] }, { "title": "Regional EKS Events", "panels": [ { "title": "EKS Events", "subtitle": null, "expressions": [ "{container=\"kube-eventrouter\",cluster_env=\"prod\",namespace=\"kube-system\",cluster=~\"prod-.*-f01\"} | json | __error__=\"\" | event_type != \"Normal\" | line_format \"ap-northeast-1 [{{.event_type}}] {{.event_involvedObject_kind}}({{.event_involvedObject_namespace}}/{{.event_involvedObject_name}}): {{.event_reason}}({{.event_message}})\"", "{container=\"kube-eventrouter\",cluster_env=\"prod\",namespace=\"kube-system\",cluster=~\"prod-.*-f01\"} | json | __error__=\"\" | event_type != \"Normal\" | line_format \"ap-southeast-1 [{{.event_type}}] {{.event_involvedObject_kind}}({{.event_involvedObject_namespace}}/{{.event_involvedObject_name}}): {{.event_reason}}({{.event_message}})\"", "{container=\"kube-eventrouter\",cluster_env=\"prod\",namespace=\"kube-system\",cluster=~\"prod-.*-f01\"} | json | __error__=\"\" | event_type != \"Normal\" | line_format \"eu-central-1 [{{.event_type}}] {{.event_involvedObject_kind}}({{.event_involvedObject_namespace}}/{{.event_involvedObject_name}}): {{.event_reason}}({{.event_message}})\"", "{container=\"kube-eventrouter\",cluster_env=\"prod\",namespace=\"kube-system\",cluster=~\"prod-.*-f01\"} | json | __error__=\"\" | event_type != \"Normal\" | line_format \"us-east-1 [{{.event_type}}] {{.event_involvedObject_kind}}({{.event_involvedObject_namespace}}/{{.event_involvedObject_name}}): {{.event_reason}}({{.event_message}})\"", "{container=\"kube-eventrouter\",cluster_env=\"prod\",namespace=\"kube-system\",cluster=~\"prod-.*-f01\"} | json | __error__=\"\" | event_type != \"Normal\" | line_format \"us-west-2 [{{.event_type}}] {{.event_involvedObject_kind}}({{.event_involvedObject_namespace}}/{{.event_involvedObject_name}}): {{.event_reason}}({{.event_message}})\"" ] } ] }, { "title": "TiDB Worker", "panels": [ { "title": "Available Scaler", "subtitle": "Available tidb-worker scaler count", "expressions": [ "kube_deployment_status_replicas_available{deployment=\"scaler\",namespace=\"tidb-worker\"}" ] }, { "title": "Available TiDB Endpoint", "subtitle": "Available tidb endpoint for meta data management", "expressions": [ "sum(kube_pod_status_ready{namespace=\"tidb-serverless\",pod=~\"serverless-cluster-tidb-.*\", condition=\"true\"})" ] }, { "title": "Worker Counts", "subtitle": "Running Worker Counts", "expressions": [ "count(kube_pod_info{namespace=\"tidb-worker\"})", "count(kube_pod_info{namespace=\"tidb-worker\",pod=~\"tidb-worker-gc.*\"})", "count(kube_pod_info{namespace=\"tidb-worker\",pod=~\"tidb-worker-ddl.*\"})", "count(kube_pod_info{namespace=\"tidb-worker\",pod=~\"tidb-worker-batch.*\"})" ] }, { "title": "GC Worker Up Time", "subtitle": "How long has gc worker been running", "expressions": [ "time() - kube_pod_start_time{namespace=\"tidb-worker\",pod=~\"tidb-worker-gc.*\"}" ] } ] }, { "title": "Vmagent", "panels": [ { "title": "Uptime", "subtitle": null, "expressions": [ "sort(time() - vm_app_start_timestamp{job=~\"vmagent-o11y.*\"})" ] }, { "title": "Vmagent Pod Status", "subtitle": null, "expressions": [ "count by (phase) (kube_pod_status_phase{namespace=\"monitoring\",pod=~\"vmagent-o11y-service-monitor-vmagent.*\"} > 0)", "count by (phase) (kube_pod_status_phase{namespace=\"monitoring\",pod=~\"vmagent-o11y-tidb-serverless-vmagent.*\"} > 0)", "increase(kube_pod_container_status_restarts_total{ namespace=\"monitoring\",container=\"vmagent\"}[1m])" ] } ] }, { "title": "Vector Agent", "panels": [ { "title": "Vector Agent Scrape Error Monitor", "subtitle": null, "expressions": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"logging-serverless\"}) by (pod, node)/\nsum(kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"logging-serverless\"}) by (pod, node) > 0.90", "sum(vector_buffer_byte_size/vector_buffer_max_byte_size > 0.2) by (host)" ] }, { "title": "Vector Agent High CPU Usage", "subtitle": null, "expressions": [ "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=\"logging-serverless\"}) by (pod, node)/\nsum(kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"logging-serverless\"}) by (pod, node) > 0.90", "sum(vector_buffer_byte_size/vector_buffer_max_byte_size > 0.2) by (host)" ] } ] } ] }