package slowquery import ( "fmt" "net/http" "strconv" "strings" "sigs.k8s.io/controller-runtime/pkg/client" "database/sql" gcpstorage "cloud.google.com/go/storage" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" "github.com/gin-gonic/gin" "github.com/pingcap/log" "github.com/pingcap/tidb/pkg/util/plancodec" "go.uber.org/zap" "github.com/tidbcloud/observability/o11ydiagnosis/pkg/slowquery/slowstats" "github.com/tidbcloud/observability/o11ydiagnosis/pkg/storage" ) const ( perClusterQPSLimit = 10.0 perClusterLimiterBucketSize = 10 maxListQueryTimeRange = 3*60*60*24 + 5*60 // 3d (+5min) maxDuckdbListQueryTimeRange = 30*60*60*24 + 5*60 // 3d (+5min) perClusterQpsLimitDuckDB = 10.0 ) type DuckDBSwitchConfig struct { DuckDBSwitchTime int DuckDBSwitchAllEnabled bool DuckDBGrayTenantIds map[string]bool Vendor string Region string } // slowMetricSeriesPoint is one time series in the JSON body for GET /slow_query/stats/metric. // nolint:unused type slowMetricSeriesPoint []int64 type SlowlogInDL struct { ID int64 `db:"id"` Time float64 `db:"time"` Digest string `db:"digest"` Date string `db:"date"` // partition column Instance sql.NullString `db:"instance"` TxnStartTS sql.NullString `db:"txn_start_ts"` // uint64 as string User sql.NullString `db:"user"` Host sql.NullString `db:"host"` ConnID sql.NullString `db:"conn_id"` // uint64 as string SessionAlias sql.NullString `db:"session_alias"` ExecRetryCount sql.NullString `db:"exec_retry_count"` // uint64 as string ExecRetryTime sql.NullFloat64 `db:"exec_retry_time"` QueryTime sql.NullFloat64 `db:"query_time"` ParseTime sql.NullFloat64 `db:"parse_time"` CompileTime sql.NullFloat64 `db:"compile_time"` RewriteTime sql.NullFloat64 `db:"rewrite_time"` PreprocSubqueries sql.NullString `db:"preproc_subqueries"` PreprocSubqueriesTime sql.NullFloat64 `db:"preproc_subqueries_time"` OptimizeTime sql.NullFloat64 `db:"optimize_time"` WaitTS sql.NullFloat64 `db:"wait_ts"` PrewriteTime sql.NullFloat64 `db:"prewrite_time"` WaitPrewriteBinlogTime sql.NullFloat64 `db:"wait_prewrite_binlog_time"` CommitTime sql.NullFloat64 `db:"commit_time"` GetCommitTSTime sql.NullFloat64 `db:"get_commit_ts_time"` CommitBackoffTime sql.NullFloat64 `db:"commit_backoff_time"` BackoffTypes sql.NullString `db:"backoff_types"` ResolveLockTime sql.NullFloat64 `db:"resolve_lock_time"` LocalLatchWaitTime sql.NullFloat64 `db:"local_latch_wait_time"` WriteKeys sql.NullInt64 `db:"write_keys"` WriteSize sql.NullInt64 `db:"write_size"` PrewriteRegion sql.NullInt64 `db:"prewrite_region"` TxnRetry sql.NullInt64 `db:"txn_retry"` CopTime sql.NullFloat64 `db:"cop_time"` ProcessTime sql.NullFloat64 `db:"process_time"` WaitTime sql.NullFloat64 `db:"wait_time"` BackoffTime sql.NullFloat64 `db:"backoff_time"` LockkeysTime sql.NullFloat64 `db:"lockkeys_time"` RequestCount sql.NullString `db:"request_count"` // uint64 as string TotalKeys sql.NullString `db:"total_keys"` // uint64 as string ProcessKeys sql.NullString `db:"process_keys"` // uint64 as string RocksdbDeleteSkippedCount sql.NullString `db:"rocksdb_delete_skipped_count"` // uint64 as string RocksdbKeySkippedCount sql.NullString `db:"rocksdb_key_skipped_count"` // uint64 as string RocksdbBlockCacheHitCount sql.NullString `db:"rocksdb_block_cache_hit_count"` // uint64 as string RocksdbBlockReadCount sql.NullString `db:"rocksdb_block_read_count"` // uint64 as string RocksdbBlockReadByte sql.NullString `db:"rocksdb_block_read_byte"` // uint64 as string DB sql.NullString `db:"db"` IndexNames sql.NullString `db:"index_names"` IsInternal sql.NullBool `db:"is_internal"` Stats sql.NullString `db:"stats"` CopProcAvg sql.NullFloat64 `db:"cop_proc_avg"` CopProcP90 sql.NullFloat64 `db:"cop_proc_p90"` CopProcMax sql.NullFloat64 `db:"cop_proc_max"` CopProcAddr sql.NullString `db:"cop_proc_addr"` CopWaitAvg sql.NullFloat64 `db:"cop_wait_avg"` CopWaitP90 sql.NullFloat64 `db:"cop_wait_p90"` CopWaitMax sql.NullFloat64 `db:"cop_wait_max"` CopWaitAddr sql.NullString `db:"cop_wait_addr"` MemMax sql.NullInt64 `db:"mem_max"` DiskMax sql.NullInt64 `db:"disk_max"` KvTotal sql.NullFloat64 `db:"kv_total"` PdTotal sql.NullFloat64 `db:"pd_total"` BackoffTotal sql.NullFloat64 `db:"backoff_total"` WriteSQLResponseTotal sql.NullFloat64 `db:"write_sql_response_total"` ResultRows sql.NullInt64 `db:"result_rows"` Warnings sql.NullString `db:"warnings"` BackoffDetail sql.NullString `db:"backoff_detail"` Prepared sql.NullBool `db:"prepared"` Succ sql.NullBool `db:"succ"` IsExplicitTxn sql.NullBool `db:"is_explicit_txn"` IsWriteCacheTable sql.NullBool `db:"is_write_cache_table"` PlanFromCache sql.NullBool `db:"plan_from_cache"` PlanFromBinding sql.NullBool `db:"plan_from_binding"` HasMoreResults sql.NullBool `db:"has_more_results"` ResourceGroup sql.NullString `db:"resource_group"` RequestUnitRead sql.NullFloat64 `db:"request_unit_read"` RequestUnitWrite sql.NullFloat64 `db:"request_unit_write"` RequestUnitV2 sql.NullFloat64 `db:"request_unit_v2"` TimeQueuedByRC sql.NullFloat64 `db:"time_queued_by_rc"` PlanDigest sql.NullString `db:"plan_digest"` PrevStmt sql.NullString `db:"prev_stmt"` ServerlessTenantID sql.NullString `db:"serverless_tenant_id"` ServerlessProjectID sql.NullString `db:"serverless_project_id"` ServerlessClusterID sql.NullString `db:"serverless_cluster_id"` WorkerRole sql.NullString `db:"worker_role"` IsTiflash sql.NullBool `db:"is_tiflash"` TidbCPUTime sql.NullFloat64 `db:"tidb_cpu_time"` TikvCPUTime sql.NullFloat64 `db:"tikv_cpu_time"` IARemoteReadSegmentCount sql.NullString `db:"ia_remote_read_segment_count"` // uint64 as string IARemoteReadSegmentSize sql.NullString `db:"ia_remote_read_segment_size"` // uint64 as string IARemoteReadSegmentWaitTime sql.NullFloat64 `db:"ia_remote_read_segment_wait_time"` RU sql.NullFloat64 `db:"ru"` RUv2 sql.NullFloat64 `db:"ru_v2"` Query sql.NullString `db:"query"` BinaryPlan sql.NullString `db:"binary_plan"` Plan sql.NullString `db:"plan"` DecodedPlan sql.NullString `db:"decoded_plan"` ReadPoolTaskDetails sql.NullString `db:"read_pool_task_details"` } var FieldCastTypeMap = map[string]string{ "txn_start_ts": "UBIGINT", "conn_id": "UBIGINT", "exec_retry_count": "UBIGINT", "preproc_subqueries": "UBIGINT", "write_keys": "UBIGINT", "write_size": "UBIGINT", "prewrite_region": "UBIGINT", "txn_retry": "UBIGINT", "request_count": "UBIGINT", "total_keys": "UBIGINT", "process_keys": "UBIGINT", "rocksdb_delete_skipped_count": "UBIGINT", "rocksdb_key_skipped_count": "UBIGINT", "rocksdb_block_cache_hit_count": "UBIGINT", "rocksdb_block_read_count": "UBIGINT", "rocksdb_block_read_byte": "UBIGINT", "ia_remote_read_segment_count": "UBIGINT", "ia_remote_read_segment_size": "UBIGINT", "mem_max": "UBIGINT", "disk_max": "UBIGINT", "result_rows": "UBIGINT", } const userAgentForceToDuckDB = "O11y-Force-DuckDB" func InitSlowQueryApi(g *gin.RouterGroup, s *Service) { g.GET("/slow_query/list", s.handleList) g.GET("/slow_query/detail", s.handleDetail) g.GET("/slow_query/databases", s.handleDatabases) g.GET("/slow_query/available_fields", s.handleAvailableFields) g.GET("/slow_query/stats", s.handleStats) g.GET("/slow_query/stats/metric", s.handleStatsMetric) g.GET("/slow_query/stats/databases", s.handleStatsDatabases) g.GET("/slow_query/stats/time_windows", s.handleStatsTimeWindows) } func NewSlowQueryService(cacheDir string, store storage.Storage, client client.Client, duckDBSwitchConfig DuckDBSwitchConfig) *Service { return &Service{ SlowQuery: newRateLimitedSlowQuery(NewSlowQuery(store), perClusterQPSLimit, perClusterLimiterBucketSize), SlowQueryV2: newRateLimitedSlowQuery(NewSlowQueryV2(store), perClusterQPSLimit, perClusterLimiterBucketSize), SlowQueryDuckDBAdapter: newRateLimitedSlowQuery(NewSlowQueryDuckDBAdapter(duckDBSwitchConfig.Vendor, duckDBSwitchConfig.Region, store, client), perClusterQpsLimitDuckDB, perClusterLimiterBucketSize), StatsQuery: slowstats.NewStatsQuery(cacheDir, store), DuckDBSwitchTime: duckDBSwitchConfig.DuckDBSwitchTime, DuckDBSwitchAllEnabled: duckDBSwitchConfig.DuckDBSwitchAllEnabled, DuckDBGrayTenantIds: duckDBSwitchConfig.DuckDBGrayTenantIds, } } // Service is a SlowQuery wrapper for handling HTTP requests and responses. type Service struct { SlowQuery SlowQuery SlowQueryV2 SlowQuery SlowQueryDuckDBAdapter SlowQuery StatsQuery *slowstats.StatsQuery DuckDBSwitchTime int DuckDBSwitchAllEnabled bool DuckDBGrayTenantIds map[string]bool } // errorResponse represents the error details returned to the web front. type errorResponse struct { Error bool `json:"error"` Message string `json:"message"` Code string `json:"code"` FullText string `json:"full_text"` } // slowLogRespItem is a SlowLog representation for HTTP transport. // // Ported from pingcap/tidb-dashboard/pkg/apiserver/slowquery/model.go type slowLogRespItem struct { Digest string `gorm:"column:Digest" json:"digest"` Query string `gorm:"column:Query" json:"query"` Instance string `gorm:"column:INSTANCE" json:"instance"` DB string `gorm:"column:DB" json:"db"` // TODO: Switch back to uint64 when modern browser as well as Swagger handles BigInt well. ConnectionID string `gorm:"column:Conn_ID" json:"connection_id"` Success int `gorm:"column:Succ" json:"success"` Timestamp float64 `gorm:"column:timestamp" proj:"(UNIX_TIMESTAMP(Time) + 0E0)" json:"timestamp"` // finish time QueryTime float64 `gorm:"column:Query_time" json:"query_time"` // latency ParseTime float64 `gorm:"column:Parse_time" json:"parse_time"` CompileTime float64 `gorm:"column:Compile_time" json:"compile_time"` RewriteTime float64 `gorm:"column:Rewrite_time" json:"rewrite_time"` PreprocSubqueriesTime float64 `gorm:"column:Preproc_subqueries_time" json:"preproc_subqueries_time"` OptimizeTime float64 `gorm:"column:Optimize_time" json:"optimize_time"` WaitTSTime float64 `gorm:"column:Wait_TS" json:"wait_ts"` CopTime float64 `gorm:"column:Cop_time" json:"cop_time"` LockKeysTime float64 `gorm:"column:LockKeys_time" json:"lock_keys_time"` WriteRespTime float64 `gorm:"column:Write_sql_response_total" json:"write_sql_response_total"` ExecRetryTime float64 `gorm:"column:Exec_retry_time" json:"exec_retry_time"` MemoryMax int `gorm:"column:Mem_max" json:"memory_max"` DiskMax int `gorm:"column:Disk_max" json:"disk_max"` // TODO: Switch back to uint64 when modern browser as well as Swagger handles BigInt well. TxnStartTS string `gorm:"column:Txn_start_ts" json:"txn_start_ts"` // Detail PrevStmt string `gorm:"column:Prev_stmt" json:"prev_stmt"` Plan string `gorm:"column:Plan" json:"plan"` // BinaryPlan string `gorm:"column:Binary_plan" json:"binary_plan"` // Basic IsInternal int `gorm:"column:Is_internal" json:"is_internal"` IndexNames string `gorm:"column:Index_names" json:"index_names"` Stats string `gorm:"column:Stats" json:"stats"` BackoffTypes string `gorm:"column:Backoff_types" json:"backoff_types"` Prepared int `json:"prepared"` PlanFromCache int `json:"plan_from_cache"` PlanFromBinding int `json:"plan_from_binding"` // Connection User string `gorm:"column:User" json:"user"` Host string `gorm:"column:Host" json:"host"` // Time ProcessTime float64 `gorm:"column:Process_time" json:"process_time"` WaitTime float64 `gorm:"column:Wait_time" json:"wait_time"` BackoffTime float64 `gorm:"column:Backoff_time" json:"backoff_time"` GetCommitTSTime float64 `gorm:"column:Get_commit_ts_time" json:"get_commit_ts_time"` LocalLatchWaitTime float64 `gorm:"column:Local_latch_wait_time" json:"local_latch_wait_time"` ResolveLockTime float64 `gorm:"column:Resolve_lock_time" json:"resolve_lock_time"` PrewriteTime float64 `gorm:"column:Prewrite_time" json:"prewrite_time"` WaitPreWriteBinlogTime float64 `gorm:"column:Wait_prewrite_binlog_time" json:"wait_prewrite_binlog_time"` CommitTime float64 `gorm:"column:Commit_time" json:"commit_time"` CommitBackoffTime float64 `gorm:"column:Commit_backoff_time" json:"commit_backoff_time"` CopProcAvg float64 `gorm:"column:Cop_proc_avg" json:"cop_proc_avg"` CopProcP90 float64 `gorm:"column:Cop_proc_p90" json:"cop_proc_p90"` CopProcMax float64 `gorm:"column:Cop_proc_max" json:"cop_proc_max"` CopWaitAvg float64 `gorm:"column:Cop_wait_avg" json:"cop_wait_avg"` CopWaitP90 float64 `gorm:"column:Cop_wait_p90" json:"cop_wait_p90"` CopWaitMax float64 `gorm:"column:Cop_wait_max" json:"cop_wait_max"` // Transaction WriteKeys int `gorm:"column:Write_keys" json:"write_keys"` WriteSize int `gorm:"column:Write_size" json:"write_size"` PrewriteRegion int `gorm:"column:Prewrite_region" json:"prewrite_region"` TxnRetry int `gorm:"column:Txn_retry" json:"txn_retry"` // Coprocessor RequestCount uint `gorm:"column:Request_count" json:"request_count"` ProcessKeys uint `gorm:"column:Process_keys" json:"process_keys"` TotalKeys uint `gorm:"column:Total_keys" json:"total_keys"` CopProcAddr string `gorm:"column:Cop_proc_addr" json:"cop_proc_addr"` CopWaitAddr string `gorm:"column:Cop_wait_addr" json:"cop_wait_addr"` // RocksDB RocksdbDeleteSkippedCount uint `gorm:"column:Rocksdb_delete_skipped_count" json:"rocksdb_delete_skipped_count"` RocksdbKeySkippedCount uint `gorm:"column:Rocksdb_key_skipped_count" json:"rocksdb_key_skipped_count"` RocksdbBlockCacheHitCount uint `gorm:"column:Rocksdb_block_cache_hit_count" json:"rocksdb_block_cache_hit_count"` RocksdbBlockReadCount uint `gorm:"column:Rocksdb_block_read_count" json:"rocksdb_block_read_count"` RocksdbBlockReadByte uint `gorm:"column:Rocksdb_block_read_byte" json:"rocksdb_block_read_byte"` // Resource Control RU float64 `gorm:"column:RU" json:"ru"` RUV2 float64 `gorm:"column:RU_v2" json:"ru_v2"` QueuedTime float64 `gorm:"column:Time_queued_by_rc" json:"time_queued_by_rc"` ResourceGroup string `gorm:"column:Resource_group" json:"resource_group"` TidbCPUTime float64 `gorm:"column:Tidb_cpu_time" json:"tidb_cpu_time"` TikvCPUTime float64 `gorm:"column:Tikv_cpu_time" json:"tikv_cpu_time"` IARemoteReadSegmentCount uint `gorm:"column:IA_remote_read_segment_count" json:"ia_remote_read_segment_count"` IARemoteReadSegmentSize uint `gorm:"column:IA_remote_read_segment_size" json:"ia_remote_read_segment_size"` IARemoteReadSegmentWaitTime float64 `gorm:"column:IA_remote_read_segment_wait_time" json:"ia_remote_read_segment_wait_time"` Warnings string `gorm:"column:Warnings" json:"warnings"` ReadPoolTaskDetails string `json:"read_pool_task_details,omitempty"` } type clusterHeader struct { Provider string `header:"x-provider" binding:"required"` Region string `header:"x-region" binding:"required"` TenantID string `header:"x-org-id"` ProjectID string `header:"x-project-id"` ClusterID string `header:"x-cluster-id" binding:"required"` DeployType string `header:"x-deploy-type"` // dedicated/shared/nextgen/premium Env string `header:"x-env"` // Use default env if emprty BizType string `header:"x-biz-type"` // byoc/other } type listRequest struct { BeginTime int `json:"begin_time" form:"begin_time"` EndTime int `json:"end_time" form:"end_time" binding:"required"` DB []string `json:"db" form:"db"` Limit int `json:"limit" form:"limit"` Text string `json:"text" form:"text"` OrderBy string `json:"orderBy" form:"orderBy"` IsDesc bool `json:"desc" form:"desc"` ShowInternal bool `json:"show_internal" form:"show_internal"` // for showing slow queries in the statement detail page Plans []string `json:"plans" form:"plans"` Digest string `json:"digest" form:"digest"` Fields string `json:"fields" form:"fields" binding:"required"` // example: "query,timestamp,query_time,memory_max,digest" } func (s *Service) switchToDuckDB(userAgent string, tenantID string, beginTime float64) bool { if strings.Contains(userAgent, userAgentForceToDuckDB) { log.Info("force to duckDB", zap.String("User Agent", userAgent)) return true } return (s.DuckDBSwitchAllEnabled || s.DuckDBGrayTenantIds[tenantID]) && s.DuckDBSwitchTime > 0 && float64(s.DuckDBSwitchTime) < beginTime } // @Summary List slow queries // @Tags slowquery // @Description Query slow query logs for a cluster within a time range, with optional filtering (digest, databases, fields). // @Produce json // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param x-deploy-type header string false "Deployment tier: dedicated/shared/nextgen/premium" // @Param begin_time query int false "Optional start time (Unix seconds, inclusive)" // @Param end_time query int true "End time (Unix seconds, inclusive)" // @Param fields query string true "Comma-separated list of fields to return (e.g. query,timestamp,query_time,memory_max,digest)" // @Param db query []string false "database names to search" // @Param limit query int false "result count limit" // @Param text query string false "Substring filter on SQL text or digest" // @Param orderBy query string false "order by which field" // @Param desc query bool false "descending order" // @Param plans query []string false "plan digests to search" // @Param digest query string false "sql digest to search" // @Param show_internal query bool false "show internal slow queries" // @Success 200 {object} []slowLogRespItem // @Router /slow_query/list [get] func (s *Service) handleList(c *gin.Context) { userAgent := c.Request.Header.Get("User-Agent") log.Debug("Received request with User-Agent", zap.String("User-Agent", userAgent)) f := recordHttpMetric(c, "/list") defer f() var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } var req listRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &ListParams{ Env: env, Provider: header.Provider, Region: header.Region, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", BeginTime: req.BeginTime, EndTime: req.EndTime, Limit: req.Limit, OrderBy: req.OrderBy, Text: req.Text, Digest: req.Digest, DB: req.DB, Plans: req.Plans, IsDesc: req.IsDesc, ShowInternal: req.ShowInternal, Fields: req.Fields, } ifSwitchToDuckDB := s.switchToDuckDB(userAgent, params.TenantID, float64(params.BeginTime)) if ifSwitchToDuckDB { if (req.EndTime - req.BeginTime) > maxDuckdbListQueryTimeRange { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "time range exceeds 30d", FullText: "time range exceeds 30d", }) return } } else { if !nextGen && (req.EndTime-req.BeginTime) > maxListQueryTimeRange { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "time range exceeds 3d", FullText: "time range exceeds 3d", }) return } } var err error var slowLogs []*SlowLog if sharedTier { slowLogs, err = s.SlowQuery.List(c.Request.Context(), params) } else { slowLogs, err = s.SlowQueryDuckDBAdapter.List(c.Request.Context(), params) } if err != nil { log.Error("failed to query slow log list", zap.Error(err)) if e, ok := err.(awserr.Error); ok && e.Code() == s3.ErrCodeNoSuchBucket { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: fmt.Sprintf("aws region '%s' not deployed", header.Region), FullText: err.Error(), }) } else if err == gcpstorage.ErrBucketNotExist { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: fmt.Sprintf("gcp region '%s' not deployed", header.Region), FullText: err.Error(), }) } else if err == ErrUnsupportedOrderBy { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: fmt.Sprintf("unsupported order by '%s'", req.OrderBy), FullText: err.Error(), }) } else { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "failed to query slow log list", FullText: err.Error(), }) } return } fields := strings.Split(req.Fields, ",") rs := make([]*slowLogRespItem, 0, len(slowLogs)) for _, sl := range slowLogs { rs = append(rs, slowLogToRespItemByFields(sl, fields)) } c.JSON(http.StatusOK, rs) } type detailRequest struct { Digest string `json:"digest" form:"digest" binding:"required"` Timestamp float64 `json:"timestamp" form:"timestamp" binding:"required"` // TODO: Switch back to uint64 when modern browser as well as Swagger handles BigInt well. ConnectID string `json:"connect_id" form:"connect_id" binding:"required"` } // @Summary Get slow query detail // @Tags slowquery // @Description Fetch the matched slow query record detail identified by digest, timestamp, and connection ID. // @Produce json // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param x-deploy-type header string false "Deployment tier: dedicated/shared/nextgen/premium" // @Param digest query string true "SQL digest (fingerprint) to match slow log item" // @Param timestamp query float64 true "Unix timestamp in seconds to match slow log item" // @Param connect_id query string true "Connection ID to match slow log item" // @Success 200 {object} slowLogRespItem // @Router /slow_query/detail [get] func (s *Service) handleDetail(c *gin.Context) { userAgent := c.Request.Header.Get("User-Agent") log.Debug("Received request with User-Agent", zap.String("User-Agent", userAgent)) f := recordHttpMetric(c, "/detail") defer f() var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } var req detailRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } connID, err := strconv.ParseInt(req.ConnectID, 10, 64) if err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: fmt.Sprintf("invalid connection id format: %s", req.ConnectID), FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &DetailParams{ Env: env, Provider: header.Provider, Region: header.Region, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", ConnID: uint64(connID), Digest: req.Digest, Timestamp: req.Timestamp, } var sl *SlowLog if sharedTier { sl, err = s.SlowQuery.Detail(c.Request.Context(), params) } else { sl, err = s.SlowQueryDuckDBAdapter.Detail(c.Request.Context(), params) } if err != nil { if e, ok := err.(awserr.Error); ok && e.Code() == s3.ErrCodeNoSuchBucket { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: fmt.Sprintf("aws region '%s' not deployed", header.Region), FullText: err.Error(), }) } else if err == gcpstorage.ErrBucketNotExist { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: fmt.Sprintf("gcp region '%s' not deployed", header.Region), FullText: err.Error(), }) } else { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "failed to query slow log detail", FullText: err.Error(), }) } return } // if len(sl.User) == 0 { // // Empty `sl.User` means this is an internal query. We don't show // // internal slow queries to the user. // c.JSON(http.StatusBadRequest, errorResponse{ // Error: true, // Message: "internal slow query", // }) // return // } resp := slowLogToRespItem(sl) c.JSON(http.StatusOK, resp) } type databasesRequest struct { BeginTime int `json:"begin_time" form:"begin_time" binding:"required"` EndTime int `json:"end_time" form:"end_time" binding:"required"` ShowInternal bool `json:"show_internal" form:"show_internal"` } // @Summary List databases with slow queries // @Tags slowquery // @Description Return the database names that have slow queries within a given time window. // @Produce json // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param x-deploy-type header string false "Deployment tier: dedicated/shared/nextgen/premium" // @Param begin_time query int true "Start time (Unix seconds, inclusive)" // @Param end_time query int true "End time (Unix seconds, inclusive)" // @Success 200 {array} string // @Router /slow_query/databases [get] func (s *Service) handleDatabases(c *gin.Context) { f := recordHttpMetric(c, "/databases") defer f() userAgent := c.Request.Header.Get("User-Agent") forceToDuckDB := strings.Contains(userAgent, userAgentForceToDuckDB) var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } var req databasesRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } beginHourTs := req.BeginTime - 72*3600 - req.BeginTime%(72*3600) sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &slowstats.ClusterInfo{ Provider: header.Provider, Region: header.Region, Env: env, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", } var databases []string var err error if nextGen || forceToDuckDB || s.switchToDuckDB(userAgent, params.TenantID, float64(req.BeginTime)) { databases, err = s.SlowQueryDuckDBAdapter.Databases(c.Request.Context(), *params, int64(req.BeginTime), (req.EndTime-req.BeginTime)/3600, req.ShowInternal) } else { databases, err = s.StatsQuery.QueryDatabases(c.Request.Context(), *params, int64(beginHourTs), 72) } if err != nil { c.JSON(http.StatusInternalServerError, errorResponse{ Error: true, Message: "failed to query databases", FullText: err.Error(), }) return } c.JSON(http.StatusOK, databases) } // @Summary List selectable fields for slow query // @Tags slowquery // @Description List the available slow query fields that can be requested via the `fields` parameter of `/slow_query/list`. Fields are filtered by the cluster's slowlog table schema. // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param x-deploy-type header string false "Deployment type: dedicated/shared/nextgen/premium" // @Param x-env header string false "Environment, default prod" // @Produce json // @Success 200 {array} string // @Router /slow_query/available_fields [get] func (s *Service) handleAvailableFields(c *gin.Context) { f := recordHttpMetric(c, "/available_fields") defer f() var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &AvailableFieldsParams{ Env: env, Provider: header.Provider, Region: header.Region, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", } var fields []string var err error if sharedTier { fields, err = s.SlowQuery.AvailableFields(c.Request.Context(), params) } else { fields, err = s.SlowQueryDuckDBAdapter.AvailableFields(c.Request.Context(), params) } if err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "failed to query available fields", FullText: err.Error(), }) return } c.JSON(http.StatusOK, fields) } type statsRequest struct { BeginTime int64 `json:"begin_time" form:"begin_time" binding:"required"` Hours int `json:"hours" form:"hours" binding:"required"` StatementTypes []string `json:"statement_types" form:"statement_types"` StatementTypesNotIn bool `json:"statement_types_not_in" form:"statement_types_not_in"` Database string `json:"database" form:"database"` // deprecated Databases []string `json:"databases" form:"databases"` Internal bool `json:"internal" form:"internal"` OrderBy string `json:"order_by" form:"order_by"` Limit int `json:"limit" form:"limit"` } // @Summary Get slow query statistics // @Tags slowquery // @Description Aggregate slow query statistics over time windows, with optional filtering by statement types and databases. // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param begin_time query int true "hour begin time in unix seconds" // @Param hours query int true "n hours (3/6/12/24/)" // @Param statement_types query []string false "Statement types to include (default: all). Example: [SELECT INSERT]" // @Param statement_types_not_in query bool false "statement types not in, default false" // @Param database query string false "database name, default empty string for all" // @Param databases query []string false "database names, default empty array for all" // @Param order_by query string false "order by field, default empty string for count" // @Param limit query int false "top limit, default 10" // @Produce json // @Success 200 {array} slowstats.SlowStats // @Router /slow_query/stats [get] func (s *Service) handleStats(c *gin.Context) { f := recordHttpMetric(c, "/stats") defer f() userAgent := c.Request.Header.Get("User-Agent") forceToDuckDB := strings.Contains(userAgent, userAgentForceToDuckDB) var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } var req statsRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if req.Database != "" { found := false for _, db := range req.Databases { if db == req.Database { found = true } } if !found { req.Databases = append(req.Databases, req.Database) } } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &slowstats.ClusterInfo{ Provider: header.Provider, Region: header.Region, Env: env, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", } var rs []slowstats.SlowStats var err error if nextGen || forceToDuckDB || s.switchToDuckDB(userAgent, params.TenantID, float64(req.BeginTime)) { rs, err = s.SlowQueryDuckDBAdapter.Stats(c.Request.Context(), *params, req.BeginTime, req.Hours, req.StatementTypes, req.StatementTypesNotIn, req.Databases, req.Internal, req.OrderBy, req.Limit) } else { rs, err = s.StatsQuery.QueryStats(c.Request.Context(), *params, req.BeginTime, req.Hours, req.StatementTypes, req.StatementTypesNotIn, req.Databases, req.Internal, req.OrderBy, req.Limit) } if err != nil { c.JSON(http.StatusInternalServerError, errorResponse{ Error: true, Message: "failed to query stats", FullText: err.Error(), }) return } c.JSON(http.StatusOK, rs) } type statsMetricRequest struct { BeginTime int64 `json:"begin_time" form:"begin_time" binding:"required"` Hours int `json:"hours" form:"hours" binding:"required"` MetricName string `json:"metric_name" form:"metric_name"` } // @Summary Get slow query statistics metric // @Tags slowquery // @Description Fetch a time series metric aggregated from slow queries. // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param begin_time query int true "Hour-aligned begin time (Unix seconds)" // @Param hours query int true "Number of hours to include (3/6/12/24)" // @Param metric_name query string false "Metric name. Example: count_per_minute (default)" // @Produce json // @Success 200 {array} slowMetricSeriesPoint // @Router /slow_query/stats/metric [get] func (s *Service) handleStatsMetric(c *gin.Context) { f := recordHttpMetric(c, "/stats/metric") defer f() userAgent := c.Request.Header.Get("User-Agent") forceToDuckDB := strings.Contains(userAgent, userAgentForceToDuckDB) var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } var req statsMetricRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &slowstats.ClusterInfo{ Provider: header.Provider, Region: header.Region, Env: env, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", } var r [][]int64 var err error if nextGen || forceToDuckDB || s.switchToDuckDB(userAgent, params.TenantID, float64(req.BeginTime)) { // Convert to ListParams for DuckDB adapter r, err = s.SlowQueryDuckDBAdapter.StatsMetric(c.Request.Context(), *params, req.BeginTime, req.Hours, req.MetricName, ) } else { r, err = s.StatsQuery.QueryMetric(c.Request.Context(), *params, req.BeginTime, req.Hours, req.MetricName) } if err != nil { c.JSON(http.StatusInternalServerError, errorResponse{ Error: true, Message: "failed to query stats", FullText: err.Error(), }) return } c.JSON(http.StatusOK, r) } type statsDatabasesRequest struct { BeginTime int64 `json:"begin_time" form:"begin_time" binding:"required"` Hours int `json:"hours" form:"hours" binding:"required"` ShowInternal bool `json:"show_internal" form:"show_internal"` } // @Summary List databases for slow query statistics // @Tags slowquery // @Description Return database names present in the aggregated slow query statistics window. // @Param x-provider header string true "Cloud provider for the query: aws or gcp" // @Param x-region header string true "Cloud region where slow logs are stored" // @Param x-org-id header string false "Organization ID in TiDB Cloud (if applicable)" // @Param x-project-id header string false "Project ID in TiDB Cloud (if applicable)" // @Param x-cluster-id header string true "Observability cluster ID for the target TiDB deployment" // @Param begin_time query int true "Hour-aligned begin time (Unix seconds)" // @Param hours query int true "Number of hours to include (3/6/12/24)" // @Produce json // @Success 200 {array} string // @Router /slow_query/stats/databases [get] func (s *Service) handleStatsDatabases(c *gin.Context) { f := recordHttpMetric(c, "/stats/databases") defer f() userAgent := c.Request.Header.Get("User-Agent") forceToDuckDB := strings.Contains(userAgent, userAgentForceToDuckDB) var header clusterHeader if err := c.ShouldBindHeader(&header); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid header", FullText: err.Error(), }) return } var req statsDatabasesRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } env := "prod" if header.Env != "" { env = header.Env } if header.DeployType == "byoc" { header.BizType = "byoc" header.DeployType = "nextgen" } beginHourTs := req.BeginTime - 72*3600 - req.BeginTime%(72*3600) sharedTier := strings.ToLower(header.DeployType) == "shared" nextGen := (strings.ToLower(header.DeployType) == "nextgen") || (strings.ToLower(header.DeployType) == "premium") params := &slowstats.ClusterInfo{ Provider: header.Provider, Region: header.Region, Env: env, TenantID: header.TenantID, ProjectID: header.ProjectID, ClusterID: header.ClusterID, SharedTier: sharedTier, NextGen: nextGen, Byoc: header.BizType == "byoc", } var rs []string var err error if nextGen || forceToDuckDB || s.switchToDuckDB(userAgent, params.TenantID, float64(req.BeginTime)) { rs, err = s.SlowQueryDuckDBAdapter.Databases(c.Request.Context(), *params, req.BeginTime, req.Hours, req.ShowInternal) } else { rs, err = s.StatsQuery.QueryDatabases(c.Request.Context(), *params, beginHourTs, 72) } if err != nil { c.JSON(http.StatusInternalServerError, errorResponse{ Error: true, Message: "failed to query stats", FullText: err.Error(), }) return } c.JSON(http.StatusOK, rs) } type statsTimeWindowsRequest struct { BeginTime int64 `json:"begin_time" form:"begin_time" binding:"required"` EndTime int64 `json:"end_time" form:"end_time" binding:"required"` Hours int64 `json:"hours" form:"hours" binding:"required"` } func (s *Service) handleStatsTimeWindows(c *gin.Context) { f := recordHttpMetric(c, "/stats/time_windows") defer f() var req statsTimeWindowsRequest if err := c.ShouldBindQuery(&req); err != nil { c.JSON(http.StatusBadRequest, errorResponse{ Error: true, Message: "invalid params", FullText: err.Error(), }) return } windows := slowstats.GetTimeWindows(req.BeginTime, req.EndTime, req.Hours) var newWindows slowstats.TimeWindows for _, w := range windows { newWindows = append(newWindows, w) } c.JSON(http.StatusOK, newWindows) } func parsePlan(plan, binaryPlan string) string { var r string var err error if binaryPlan != "" && strings.HasPrefix(binaryPlan, "tidb_decode_binary_plan('") { binaryPlan = strings.TrimPrefix(binaryPlan, "tidb_decode_binary_plan('") binaryPlan = strings.TrimSuffix(binaryPlan, "')") r, err = plancodec.DecodeBinaryPlan(binaryPlan) } else if plan != "" && strings.HasPrefix(plan, "tidb_decode_plan('") { plan = strings.TrimPrefix(plan, "tidb_decode_plan('") plan = strings.TrimSuffix(plan, "')") r, err = plancodec.DecodePlan(plan) } else { r = plan } if err != nil { log.Error("decode plan in slow log failed", zap.String("plan", plan), zap.String("binaryPlan", binaryPlan), zap.Error(err)) } return r } func computeRUv2(sl *SlowLog) float64 { // Prefer the raw RU_v2 if present; otherwise compute from request_unit_read/write_v2. if sl.RUv2 != 0 { return sl.RUv2 } return sl.RequestUnitV2 } // //func vmAddr(env string) string { // switch strings.ToLower(env) { // case "staging": // return "http://www.ds.us-west-2.aws.observability-staging.pingcap.cloud" // case "prod": // return "http://www.ds.us-west-2.aws.observability.tidbcloud.com" // case "dev": // fallthrough // default: // return "http://www.ds.us-west-2.aws.observability-dev.pingcap.cloud" // } //} func slowLogToRespItem(sl *SlowLog) *slowLogRespItem { resp := &slowLogRespItem{ Digest: sl.Digest, Query: sl.Query, Instance: sl.Instance, DB: sl.DB, ConnectionID: fmt.Sprintf("%d", sl.ConnID), Success: boolToInt(sl.Succ), Timestamp: timeToFloat(sl.Time), QueryTime: sl.QueryTime, ParseTime: sl.ParseTime, CompileTime: sl.CompileTime, RewriteTime: sl.RewriteTime, PreprocSubqueriesTime: sl.PreprocSubqueriesTime, OptimizeTime: sl.OptimizeTime, WaitTSTime: sl.WaitTs, CopTime: sl.CopTime, LockKeysTime: sl.LockKeysTime, WriteRespTime: sl.WriteSQLResponseTotal, ExecRetryTime: sl.ExecRetryTime, MemoryMax: int(sl.MemMax), DiskMax: int(sl.DiskMax), TxnStartTS: fmt.Sprintf("%d", sl.TxnStartTs), PrevStmt: sl.PrevStmt, Plan: parsePlan(sl.Plan, sl.BinaryPlan), IsInternal: boolToInt(sl.IsInternal), IndexNames: sl.IndexNames, Stats: sl.Stats, BackoffTypes: sl.BackoffTypes, Prepared: boolToInt(sl.Prepared), PlanFromCache: boolToInt(sl.PlanFromCache), PlanFromBinding: boolToInt(sl.PlanFromBinding), User: sl.User, Host: sl.Host, ProcessTime: sl.ProcessTime, WaitTime: sl.WaitTime, BackoffTime: sl.BackoffTime, GetCommitTSTime: sl.GetCommitTsTime, LocalLatchWaitTime: sl.LocalLatchWaitTime, ResolveLockTime: sl.ResolveLockTime, PrewriteTime: sl.PrewriteTime, WaitPreWriteBinlogTime: sl.WaitPrewriteBinlogTime, CommitTime: sl.CommitTime, CommitBackoffTime: sl.CommitBackoffTime, CopProcAvg: sl.CopProcAvg, CopProcP90: sl.CopProcP90, CopProcMax: sl.CopProcMax, CopWaitAvg: sl.CopWaitAvg, CopWaitP90: sl.CopWaitP90, CopWaitMax: sl.CopWaitMax, WriteKeys: int(sl.WriteKeys), WriteSize: int(sl.WriteSize), PrewriteRegion: int(sl.PrewriteRegion), TxnRetry: int(sl.TxnRetry), RequestCount: uint(sl.RequestCount), ProcessKeys: uint(sl.ProcessKeys), TotalKeys: uint(sl.TotalKeys), CopProcAddr: sl.CopProcAddr, CopWaitAddr: sl.CopWaitAddr, RocksdbDeleteSkippedCount: uint(sl.RocksdbDeleteSkippedCount), RocksdbKeySkippedCount: uint(sl.RocksdbKeySkippedCount), RocksdbBlockCacheHitCount: uint(sl.RocksdbBlockCacheHitCount), RocksdbBlockReadCount: uint(sl.RocksdbBlockReadCount), RocksdbBlockReadByte: uint(sl.RocksdbBlockReadByte), RU: sl.RequestUnitRead + sl.RequestUnitWrite, RUV2: computeRUv2(sl), QueuedTime: sl.TimeQueuedByRc, ResourceGroup: sl.ResourceGroup, TidbCPUTime: sl.TidbCPUTime, TikvCPUTime: sl.TikvCPUTime, IARemoteReadSegmentCount: uint(sl.IARemoteReadSegmentCount), IARemoteReadSegmentSize: uint(sl.IARemoteReadSegmentSize), IARemoteReadSegmentWaitTime: sl.IARemoteReadSegmentWaitTime, Warnings: sl.Warnings, ReadPoolTaskDetails: sl.ReadPoolTaskDetails, } // resp.BinaryPlan, _ = binaryplan.GenerateBinaryPlanJSON(resp.BinaryPlan) return resp } func slowLogToRespItemByFields(sl *SlowLog, fields []string) *slowLogRespItem { item := &slowLogRespItem{ Digest: sl.Digest, ConnectionID: fmt.Sprintf("%d", sl.ConnID), Timestamp: timeToFloat(sl.Time), } for _, field := range fields { switch field { // case "digest": // item.Digest = sl.Digest case "query": item.Query = sl.Query case "instance": item.Instance = sl.Instance case "db": item.DB = sl.DB // case "connection_id": // item.ConnectionID = fmt.Sprintf("%d", sl.ConnID) case "success": item.Success = boolToInt(sl.Succ) // case "timestamp": // item.Timestamp = timeToFloat(sl.Time) case "query_time": item.QueryTime = sl.QueryTime case "parse_time": item.ParseTime = sl.ParseTime case "compile_time": item.CompileTime = sl.CompileTime case "rewrite_time": item.RewriteTime = sl.RewriteTime case "preproc_subqueries_time": item.PreprocSubqueriesTime = sl.PreprocSubqueriesTime case "optimize_time": item.OptimizeTime = sl.OptimizeTime case "wait_ts": item.WaitTSTime = sl.WaitTs case "cop_time": item.CopTime = sl.CopTime case "lock_keys_time": item.LockKeysTime = sl.LockKeysTime case "write_sql_response_total": item.WriteRespTime = sl.WriteSQLResponseTotal case "exec_retry_time": item.ExecRetryTime = sl.ExecRetryTime case "memory_max": item.MemoryMax = int(sl.MemMax) case "disk_max": item.DiskMax = int(sl.DiskMax) case "txn_start_ts": item.TxnStartTS = fmt.Sprintf("%d", sl.TxnStartTs) case "prev_stmt": item.PrevStmt = sl.PrevStmt case "plan": item.Plan = parsePlan(sl.Plan, sl.BinaryPlan) case "read_pool_task_details": item.ReadPoolTaskDetails = sl.ReadPoolTaskDetails case "is_internal": item.IsInternal = boolToInt(sl.IsInternal) case "index_names": item.IndexNames = sl.IndexNames case "stats": item.Stats = sl.Stats case "backoff_types": item.BackoffTypes = sl.BackoffTypes case "prepared": item.Prepared = boolToInt(sl.Prepared) case "plan_from_cache": item.PlanFromCache = boolToInt(sl.PlanFromCache) case "plan_from_biniding": item.PlanFromBinding = boolToInt(sl.PlanFromBinding) case "user": item.User = sl.User case "host": item.Host = sl.Host case "process_time": item.ProcessTime = sl.ProcessTime case "wait_time": item.WaitTime = sl.WaitTime case "backoff_time": item.BackoffTime = sl.BackoffTime case "get_commit_ts_time": item.GetCommitTSTime = sl.GetCommitTsTime case "local_latch_wait_time": item.LocalLatchWaitTime = sl.LocalLatchWaitTime case "resolve_lock_time": item.ResolveLockTime = sl.ResolveLockTime case "prewrite_time": item.PrewriteTime = sl.PrewriteTime case "wait_prewrite_binlog_time": item.WaitPreWriteBinlogTime = sl.WaitPrewriteBinlogTime case "commit_time": item.CommitTime = sl.CommitTime case "commit_backoff_time": item.CommitBackoffTime = sl.CommitBackoffTime case "cop_proc_avg": item.CopProcAvg = sl.CopProcAvg case "cop_proc_p90": item.CopProcP90 = sl.CopProcP90 case "cop_proc_max": item.CopProcMax = sl.CopProcMax case "cop_wait_avg": item.CopWaitAvg = sl.CopWaitAvg case "cop_wait_p90": item.CopWaitP90 = sl.CopWaitP90 case "cop_wait_max": item.CopWaitMax = sl.CopWaitMax case "write_keys": item.WriteKeys = int(sl.WriteKeys) case "write_size": item.WriteSize = int(sl.WriteSize) case "prewrite_region": item.PrewriteRegion = int(sl.PrewriteRegion) case "txn_retry": item.TxnRetry = int(sl.TxnRetry) case "request_count": item.RequestCount = uint(sl.RequestCount) case "process_keys": item.ProcessKeys = uint(sl.ProcessKeys) case "total_keys": item.TotalKeys = uint(sl.TotalKeys) case "cop_proc_addr": item.CopProcAddr = sl.CopProcAddr case "cop_wait_addr": item.CopWaitAddr = sl.CopWaitAddr case "rocksdb_delete_skipped_count": item.RocksdbDeleteSkippedCount = uint(sl.RocksdbDeleteSkippedCount) case "rocksdb_key_skipped_count": item.RocksdbKeySkippedCount = uint(sl.RocksdbKeySkippedCount) case "rocksdb_block_cache_hit_count": item.RocksdbBlockCacheHitCount = uint(sl.RocksdbBlockCacheHitCount) case "rocksdb_block_read_count": item.RocksdbBlockReadCount = uint(sl.RocksdbBlockReadCount) case "rocksdb_block_read_byte": item.RocksdbBlockReadByte = uint(sl.RocksdbBlockReadByte) case "ru": item.RU = sl.RU case "ru_v2": item.RUV2 = computeRUv2(sl) case "time_queued_by_rc": item.QueuedTime = sl.TimeQueuedByRc case "resource_group": item.ResourceGroup = sl.ResourceGroup case "tidb_cpu_time": item.TidbCPUTime = sl.TidbCPUTime case "tikv_cpu_time": item.TikvCPUTime = sl.TikvCPUTime case "ia_remote_read_segment_count": item.IARemoteReadSegmentCount = uint(sl.IARemoteReadSegmentCount) case "ia_remote_read_segment_size": item.IARemoteReadSegmentSize = uint(sl.IARemoteReadSegmentSize) case "ia_remote_read_segment_wait_time": item.IARemoteReadSegmentWaitTime = sl.IARemoteReadSegmentWaitTime case "warnings": item.Warnings = sl.Warnings } } return item } func boolToInt(v bool) int { if v { return 1 } return 0 } //func isValidTimePeriod(beginTime, endTime int) bool { // t1 := time.Unix(int64(beginTime), 0) // t2 := time.Unix(int64(endTime), 0) // // if t1.Minute() == 0 && t1.Second() == 0 { // if t2.Minute() == 0 && t2.Second() == 0 { // if endTime-beginTime == 60*60 { // return true // } else if endTime-beginTime == 24*60*60 { // if t1.Hour() == 0 && t2.Hour() == 0 { // return true // } // } // } // } // return false //}