package cloud

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httputil"
	"net/url"
	url2 "net/url"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/dchest/uniuri"
	"github.com/form3tech-oss/jwt-go"
	"github.com/gin-gonic/gin"
	"github.com/patrickmn/go-cache"
	"github.com/pingcap/errors"
	"github.com/pingcap/log"
	prometheusC "github.com/prometheus/client_golang/api"
	prometheusAPI "github.com/prometheus/client_golang/api/prometheus/v1"
	prometheusModel "github.com/prometheus/common/model"
	"go.uber.org/zap"

	"github.com/pingcap-inc/clinic/pkg/api/common"
	error2 "github.com/pingcap-inc/clinic/pkg/api/error"
	"github.com/pingcap-inc/clinic/pkg/auth/session"
	"github.com/pingcap-inc/clinic/pkg/config"
	"github.com/pingcap-inc/clinic/pkg/datasource"
	"github.com/pingcap-inc/clinic/pkg/datasource/clustersearch"
	"github.com/pingcap-inc/clinic/pkg/datasource/view"
	"github.com/pingcap-inc/clinic/pkg/db"
	"github.com/pingcap-inc/clinic/pkg/db/model"
	"github.com/pingcap-inc/clinic/pkg/importer/metrics"
	"github.com/pingcap-inc/clinic/pkg/stream"
)

type listOrganizationAndClusterFunc func(
	*session.CtxUser,
	*view.ListOrganizationAndClusterFilter,
	int,
	int,
	bool,
) (*datasource.ListOrganizationAndClusterResult, error)

type GlobalDashboardService struct {
	cfg                        *config.Config
	secret                     []byte
	sessionStore               *cache.Cache
	lock                       sync.Mutex
	listOrganizationAndCluster listOrganizationAndClusterFunc
}

func NewGlobalService(cfg *config.Config) *GlobalDashboardService {
	return &GlobalDashboardService{
		cfg:                        cfg,
		lock:                       sync.Mutex{},
		sessionStore:               cache.New(time.Minute*30, time.Minute*40),
		secret:                     []byte(cfg.DBaasConfig.DashboardConfig.SessionSecret),
		listOrganizationAndCluster: datasource.ListOrganizationAndClusterPage,
	}
}

// @Summary		list cloud clusters status
// @Tags			GlobalDashboard
// @Description	list cloud clusters status
// @Accept			json
// @Param			query			query		string		false	"search data from tenant id, tenant name, cluster id and cluster name"
// @Param			show_deleted	query		bool		false	"show deleted cluster"
// @Param			cluster_id		query		[]string	false	"cluster id"		collectionFormat(multi)
// @Param			cluster_status	query		[]string	false	"cluster status"	collectionFormat(multi)
// @Param			cluster_version	query		[]string	false	"cluster version"	collectionFormat(multi)
// @Param			deploy_type		query		[]string	false	"deploy type"		collectionFormat(multi)
// @Param			deploy_type_v2	query		[]string	false	"deploy type v2"	collectionFormat(multi)
// @Param			tenant_plan		query		[]string	false	"tenant plan"		collectionFormat(multi)
// @Param			control_plane	query		[]string	false	"control plane"		collectionFormat(multi)
// @Param			provider		query		[]string	false	"cloud provider"	collectionFormat(multi)
// @Param			region			query		[]string	false	"cloud region"		collectionFormat(multi)
// @Param			sort			query		string		false	"sort"
// @Param			order			query		string		false	"order"
// @Param			limit			query		int			false	"the count per page"
// @Param			page			query		int			false	"the page index"
// @Param			include_total	query		bool		false	"include the exact total" default(false)
// @Param			x-csrf-token	header		string		true	"get value from login.ValidationResp response"
// @Success		200				{object}	DashboardResp
// @Router			/dashboard/clusters [get]
func (gd *GlobalDashboardService) CloudClusters(c *gin.Context) {
	u := session.GetUser(c)
	if u == nil {
		_ = c.Error(error2.ErrNotFound.New("the user is not found"))
		return
	}

	pq := common.ParsePaginationQuery(c.Request.URL.RequestURI())
	query := c.Query("query")
	includeTotal, err := parseDashboardIncludeTotal(c)
	if err != nil {
		_ = c.Error(error2.ErrBadRequest.New("invalid include_total"))
		return
	}

	result, err := gd.listOrganizationAndCluster(u, &view.ListOrganizationAndClusterFilter{
		Query:            query,
		Hide_deleted:     c.Query("show_deleted") == "false",
		ClusterIDs:       c.QueryArray("cluster_id"),
		ClusterStatus:    c.QueryArray("cluster_status"),
		Version:          c.QueryArray("cluster_version"),
		DeployType:       c.QueryArray("deploy_type"),
		DeployTypeV2:     c.QueryArray("deploy_type_v2"),
		TenantPlan:       c.QueryArray("tenant_plan"),
		ControlPlaneArch: c.QueryArray("control_plane"),
		Provider:         c.QueryArray("provider"),
		Region:           c.QueryArray("region"),
		Sort:             c.Query("sort"),
		Order:            c.Query("order"),
	}, pq.Page, pq.Limit, includeTotal)
	if err != nil {
		if err == datasource.ErrInvalidDashboardPagination {
			_ = c.Error(error2.ErrBadRequest.New("invalid pagination"))
			return
		}
		if err == clustersearch.ErrTextSearchPolicyUnavailable {
			_ = c.Error(error2.ErrServiceUnavailable.New("%s", err.Error()))
			return
		}
		_ = c.Error(error2.ErrInternal.New("fetch data failed"))
		return
	}

	c.JSON(http.StatusOK, newDashboardResp(result, includeTotal))
}

// @Summary		list regions
// @Tags			GlobalDashboard
// @Description	list regions
// @Accept			json
// @Param			provider		query		[]string	false	"cloud provdier, default for all providers"
// @Param			x-csrf-token	header		string		true	"get value from login.ValidationResp response"
// @Success		200				{object}	RegionsResp
// @Router			/dashboard/regions [get]
func (gd *GlobalDashboardService) GetRegions(c *gin.Context) {
	u := session.GetUser(c)
	if u == nil {
		_ = c.Error(error2.ErrNotFound.New("the user is not found"))
		return
	}
	providers := c.QueryArray("provider")
	regions, err := datasource.ListRegions(u, providers)
	if err != nil {
		log.Error("failed to list regions", zap.Strings("providers", providers), zap.Error(err))
		_ = c.Error(error2.ErrInternal.New("failed to list regions"))
		return
	}

	c.JSON(http.StatusOK, RegionsResp{
		Total: len(regions),
		Items: regions,
	})
}

// @Summary		list cloud clusters status form local db
// @Tags			GlobalDashboard
// @Description	list cloud clusters status form local db
// @Accept			json
// @Param			query			query		string		false	"search data from tenant id, tenant name, cluster id and cluster name"
// @Param			cluster_id		query		[]string	false	"cluster id"		collectionFormat(multi)
// @Param			cluster_status	query		[]string	false	"cluster status"	collectionFormat(multi)
// @Param			cluster_version	query		[]string	false	"cluster version"	collectionFormat(multi)
// @Param			deploy_type		query		[]string	false	"deploy type"		collectionFormat(multi)
// @Param			deploy_type_v2	query		[]string	false	"deploy type v2"	collectionFormat(multi)
// @Param			tenant_plan		query		[]string	false	"tenant plan"		collectionFormat(multi)
// @Param			show_deleted	query		bool		false	"show deleted cluster"
// @Param			control_plane	query		[]string	false	"control plane"		collectionFormat(multi)
// @Param			provider		query		[]string	false	"cloud provider"	collectionFormat(multi)
// @Param			region			query		[]string	false	"cloud region"		collectionFormat(multi)
// @Param			created_ts_from	query		uint64		false	"created_at timestamp from"
// @Param			created_ts_to	query		uint64		false	"created_at timestamp to"
// @Param			deleted_ts_from	query		uint64		false	"deleted_at timestamp from"
// @Param			deleted_ts_to	query		uint64		false	"deleted_at timestamp to"
// @Param			sort			query		string		false	"sort"
// @Param			order			query		string		false	"order"
// @Param			limit			query		int			false	"the count per page"
// @Param			page			query		int			false	"the page index"
// @Param			include_total	query		bool		false	"include the exact total" default(false)
// @Param			x-csrf-token	header		string		true	"get value from login.ValidationResp response"
// @Success		200				{object}	DashboardResp
// @Router			/dashboard/clusters2 [get]
func (gd *GlobalDashboardService) CloudClusters2(c *gin.Context) {
	u := session.GetUser(c)
	if u == nil {
		_ = c.Error(error2.ErrNotFound.New("the user is not found"))
		return
	}

	pq := common.ParsePaginationQuery(c.Request.URL.RequestURI())
	query := c.Query("query")
	includeTotal, err := parseDashboardIncludeTotal(c)
	if err != nil {
		_ = c.Error(error2.ErrBadRequest.New("invalid include_total"))
		return
	}

	createdAtFrom := int64(0)
	if createdAtFromStr := c.Query("created_ts_from"); createdAtFromStr != "" {
		createdAtFrom, err = strconv.ParseInt(createdAtFromStr, 10, 64)
		if err != nil {
			_ = c.Error(error2.ErrBadRequest.New("invalid created_ts_from"))
			return
		}
	}
	createdAtTo := int64(0)
	if createdAtToStr := c.Query("created_ts_to"); createdAtToStr != "" {
		createdAtTo, err = strconv.ParseInt(createdAtToStr, 10, 64)
		if err != nil {
			_ = c.Error(error2.ErrBadRequest.New("invalid created_ts_to"))
			return
		}
	}
	deletedAtFrom := int64(0)
	if deletedAtFromStr := c.Query("deleted_ts_from"); deletedAtFromStr != "" {
		deletedAtFrom, err = strconv.ParseInt(deletedAtFromStr, 10, 64)
		if err != nil {
			_ = c.Error(error2.ErrBadRequest.New("invalid deleted_ts_from"))
			return
		}
	}
	deletedAtTo := int64(0)
	if deletedAtToStr := c.Query("deleted_ts_to"); deletedAtToStr != "" {
		deletedAtTo, err = strconv.ParseInt(deletedAtToStr, 10, 64)
		if err != nil {
			_ = c.Error(error2.ErrBadRequest.New("invalid deleted_ts_to"))
			return
		}
	}

	result, err := gd.listOrganizationAndCluster(u, &view.ListOrganizationAndClusterFilter{
		Query:            query,
		ClusterIDs:       c.QueryArray("cluster_id"),
		ClusterStatus:    c.QueryArray("cluster_status"),
		Version:          c.QueryArray("cluster_version"),
		DeployType:       c.QueryArray("deploy_type"),
		DeployTypeV2:     c.QueryArray("deploy_type_v2"),
		TenantPlan:       c.QueryArray("tenant_plan"),
		Hide_deleted:     c.Query("show_deleted") == "false",
		ControlPlaneArch: c.QueryArray("control_plane"),
		Provider:         c.QueryArray("provider"),
		Region:           c.QueryArray("region"),
		CreatedAtFrom:    createdAtFrom,
		CreatedAtTo:      createdAtTo,
		DeletedAtFrom:    deletedAtFrom,
		DeletedAtTo:      deletedAtTo,
		Sort:             c.Query("sort"),
		Order:            c.Query("order"),
	}, pq.Page, pq.Limit, includeTotal)
	if err != nil {
		if err == datasource.ErrInvalidDashboardPagination {
			_ = c.Error(error2.ErrBadRequest.New("invalid pagination"))
			return
		}
		if err == clustersearch.ErrTextSearchPolicyUnavailable {
			_ = c.Error(error2.ErrServiceUnavailable.New("%s", err.Error()))
			return
		}
		log.Error("fetch clusters failed", zap.String("queryStr", query), zap.Error(err))
		_ = c.Error(error2.ErrInternal.New("fetch data failed"))
		return
	}

	c.JSON(http.StatusOK, newDashboardResp(result, includeTotal))
}

func parseDashboardIncludeTotal(c *gin.Context) (bool, error) {
	raw, exists := c.GetQuery("include_total")
	if !exists {
		return false, nil
	}
	return strconv.ParseBool(raw)
}

func newDashboardResp(result *datasource.ListOrganizationAndClusterResult, includeTotal bool) DashboardResp {
	resp := DashboardResp{Items: result.Items}
	if includeTotal {
		resp.Total = result.Total
		return resp
	}
	hasMore := result.HasMore
	resp.HasMore = &hasMore
	return resp
}

// @Summary		proxy dashboard
// @Tags			GlobalDashboard
// @Description	proxy dashboard
// @Accept			json
// @Param			host			query	string	true	"the proxy host"
// @Param			x-csrf-token	header	string	true	"get value from login.ValidationResp response"
// @Success		200				{array}	DashboardResp
// @Router			/dashboard/proxy [get]
func (gd *GlobalDashboardService) Proxy(c *gin.Context) {
	u := session.GetUser(c)
	if u == nil {
		_ = c.Error(error2.ErrNotFound.New("the user is not found"))
		return
	}

	if !u.Internal {
		_ = c.Error(error2.ErrForbidden.New("permission deny"))
		return
	}

	token, err := gd.getToken(u)
	if err != nil {
		_ = c.Error(error2.ErrInternal.New("get configuration failed, please try again"))
		return
	}

	orgID := c.Request.Header.Get("x-org-id")
	clusterID := c.Request.Header.Get("x-cluster-id")
	enabled, err := datasource.CloudClusterIsUserDebugAuthEnable(u, orgID)
	if err != nil {
		_ = c.Error(error2.ErrNotFound.New("%s", err.Error()))
		return
	}

	if enabled {
		_, err := datasource.CloudClusterIsAccessDashboardAuthorized(u, orgID, clusterID, u.Username)
		if err != nil {
			_ = c.Error(error2.ErrNotFound.New("%s", err.Error()))
			return
		}
	}

	url := &url2.URL{
		Scheme: "https",
		Host:   fmt.Sprintf("%s:443", gd.cfg.DBaasConfig.DashboardConfig.Endpoint),
	}
	proxy := httputil.NewSingleHostReverseProxy(url)
	originDirector := proxy.Director
	director := func(req *http.Request) {
		req.AddCookie(&http.Cookie{
			Name:     "debug.session_id",
			Value:    *token,
			Path:     "/",
			Domain:   "",
			MaxAge:   int(corePortalSessionTimeout / time.Second),
			SameSite: http.SameSiteNoneMode,
			Secure:   true,
			HttpOnly: true,
		})
		req.Host = gd.cfg.DBaasConfig.DashboardConfig.Endpoint
		req.URL.Host = gd.cfg.DBaasConfig.DashboardConfig.Endpoint
		req.URL.Scheme = "https"
		req.URL.Path = strings.TrimPrefix(req.URL.Path, "/clinic/api/v1/dashboard")
		originDirector(req)
	}
	proxy.Director = director

	proxy.ServeHTTP(c.Writer, c.Request)
}

func (gs *GlobalDashboardService) getToken(u *session.CtxUser) (*string, error) {
	v, ok := gs.sessionStore.Get("token")
	if ok {
		return v.(*string), nil
	}

	gs.lock.Lock()
	defer gs.lock.Unlock()

	key, err := gs.corePortalSession(&CorePortalSession{
		Email:     u.Email,
		Username:  u.Username,
		CSRFToken: uniuri.New(),
	})
	if err != nil {
		log.Error("new session failed", zap.Error(err))
		return nil, errors.New("get configuration failed, please try again")
	}

	gs.sessionStore.Set("token", &key, time.Minute*30)

	return &key, nil
}

const (
	corePortalSessionTimeout = time.Hour * 1
	secondsPerDay            = 60 * 60 * 24
)

func (gs *GlobalDashboardService) corePortalSession(session *CorePortalSession) (string, error) {
	tokenBody := CorePortalSessionTokenBody{
		Session: session,
		StandardClaims: jwt.StandardClaims{
			ExpiresAt: time.Now().Add(corePortalSessionTimeout).Unix(),
		},
	}
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, tokenBody)
	tokenString, err := token.SignedString(gs.secret)
	if err != nil {
		return "", err
	}
	return tokenString, nil
}

type CorePortalSession struct {
	Email     string `json:"email"`
	Username  string `json:"username"`
	CSRFToken string `json:"csrfToken"`
}

type CorePortalSessionTokenBody struct {
	jwt.StandardClaims
	Session *CorePortalSession `json:"user"`
}

var reportTypes = map[string]string{
	"workload": "System Check Report",
	"health2":  "Bechmark Report",
	"infra":    "Infra Report",
}

func (rs *GlobalDashboardService) ReportLinks(c *gin.Context) {
	resp := make([]*ListReportLink, 0)

	host := c.Request.Host
	if rs.cfg.ServerConfig.Host != "" {
		host = rs.cfg.ServerConfig.Host
	}

	stream.OfMap(reportTypes).Map(func(entry stream.MapEntry) *ListReportLink {
		reports, total, err := model.QueryReportsByAdmin(entry.Key.String(), 0, 100)
		if err != nil {
			log.Error("list reports failed", zap.Error(err))
			return nil
		}

		links := make([]string, 0)
		stream.OfSlice(reports).Map(func(r db.Report) string {
			return fmt.Sprintf("https://%s/portal/#/orgs/%d/clusters/%s/benchmark-report/%s", host, r.OrgID, r.ClusterID, r.UUID)
		}).CollectEmptySlice(&links)

		return &ListReportLink{
			Type:  entry.Value.String(),
			Total: total,
			Links: links,
		}
	}).Filter(func(link *ListReportLink) bool {
		return link != nil
	}).CollectEmptySlice(&resp)

	// comparison report
	comparisonReports, total, err := model.QueryComparisonReportByAdmin(0, 100)
	if err != nil {
		log.Error("list reports failed", zap.Error(err))
		_ = c.Error(error2.ErrInternal.New("get ComparisonReport failed"))
		return
	}

	links := make([]string, 0)
	stream.OfSlice(comparisonReports).Map(func(r db.ComparisonReport) string {
		return fmt.Sprintf("https://%s/portal/#/orgs/%d/clusters/%s/benchmark-report/%d", host, r.OrgID, r.ClusterID, r.ID)
	}).CollectEmptySlice(&links)

	resp = append(resp, &ListReportLink{
		Type:  "ComparisonReport",
		Total: total,
		Links: links,
	})

	c.JSON(http.StatusOK, ListReportLinkResponse{
		Reports: resp,
	})
}

type ActiveQPSResp struct {
	Total int            `json:"total"`
	Items []*clusterInfo `json:"items"`
}

type clusterInfo struct {
	OrgID       string  `json:"org_id"` // clinic org_id
	Region      string  `json:"region"`
	TenantID    string  `json:"tenant_id"`
	TenantName  string  `json:"tenant_name"`
	ClusterID   string  `json:"cluster_id"`
	ClusterName string  `json:"cluster_name"`
	DeployType  string  `json:"deploy_type"`
	Timestamp   int64   `json:"timestamp"`
	Duration    float64 `json:"duration"`
	QPS         float64 `json:"max_qps"`
	ReportUUID  *string `json:"report_uuid,omitempty"`
}

type queryResult struct {
	Type   prometheusModel.ValueType `json:"resultType"`
	Result any                       `json:"result"`

	// The decoded value.
	prometheusModel.Value
}

// @Summary		list cloud clusters QPS status
// @Tags			GlobalDashboard
// @Description	list cloud clusters QPS status
// @Accept			json
// @Param			threshold		query		string		true	"threshold"
// @Param			search			query		string		false	"search tenant name/id or cluster name/id"
// @Param			region			query		[]string	false	"cluster region filter"	collectionFormat(multi)
// @Param			deploy_type		query		[]string	false	"deploy type filter"	collectionFormat(multi)
// @Param			begin_ts		query		int			true	"begin_ts"
// @Param			end_ts			query		int			true	"end_ts"
// @Param			x-csrf-token	header		string		true	"get value from login.ValidationResp response"
// @Success		200				{object}	ActiveQPSResp
// @Router			/dashboard/activeqps [get]
func (gd *GlobalDashboardService) GetActiveQPS(c *gin.Context) {
	u := session.GetUser(c)
	if u == nil {
		_ = c.Error(error2.ErrNotFound.New("the user is not found"))
		return
	}

	if !u.Internal {
		_ = c.Error(error2.ErrForbidden.New("permission deny"))
		return
	}

	endTimeStr := c.Query("end_ts")
	if endTimeStr == "" {
		_ = c.Error(error2.ErrBadRequest.New("empty end_ts"))
		return
	}
	endTimeInt, err := strconv.ParseInt(endTimeStr, 10, 64)
	if err != nil {
		_ = c.Error(error2.ErrBadRequest.New("invalid end_ts"))
		return
	}
	if endTimeInt%secondsPerDay != secondsPerDay-1 {
		// format endtime to last second of a UTC+0 day
		endTimeInt = (endTimeInt/secondsPerDay+1)*secondsPerDay - 1
	}
	beginTimeStr := c.Query("begin_ts")
	if beginTimeStr == "" {
		_ = c.Error(error2.ErrBadRequest.New("empty begin_ts"))
		return
	}
	beginTimeInt, err := strconv.ParseInt(beginTimeStr, 10, 64)
	if err != nil {
		_ = c.Error(error2.ErrBadRequest.New("invalid begin_ts"))
		return
	}
	timeRangeDays := (endTimeInt - beginTimeInt) / secondsPerDay
	threshold := c.Query("threshold")
	if threshold == "" {
		_ = c.Error(error2.ErrBadRequest.New("empty threshold"))
		return
	}

	search := c.Query("search")
	region := c.QueryArray("region")
	deployType := c.QueryArray("deploy_type")

	metricConfig := gd.cfg.DiagnosisConfig.MetricConfig.SelectAddr
	// QPS metrics are stored at org 8899
	queryURL := fmt.Sprintf("http://%s:%d/select/8899/prometheus/api/v1/query", metricConfig.Host, metricConfig.Port)
	client := &http.Client{Timeout: time.Second * 60}

	tmp := make(map[string]*clusterInfo)
	resp := ActiveQPSResp{Total: 0, Items: make([]*clusterInfo, 0)}

	// store result of qps metric to a map
	query := fmt.Sprintf("%s[%dd]", getQPSQueryWithSearch("max_over_time(clinic_diagnosis_active_qps_max%s)", threshold, search, region, deployType), timeRangeDays)
	log.Info("active qps max promQL:" + query)
	result, err := client.PostForm(
		queryURL,
		url.Values{
			"query": {query},
			"time":  {fmt.Sprintf("%d", endTimeInt)},
			"step":  {"1d"}, // only usable for virtorametric
		},
	)
	if err != nil {
		log.Error(err.Error())
		_ = c.Error(error2.ErrInternal.New("internal error"))
	}
	defer result.Body.Close()
	var qpsResp metrics.PromQueryResponse
	d := json.NewDecoder(result.Body)
	err = d.Decode(&qpsResp)
	if err != nil {
		log.Error("decode active aps query result failed")
		_ = c.Error(error2.ErrInternal.New("internal error"))
		return
	}
	for _, item := range qpsResp.Data.Result {
		for _, qps := range item.Values {
			tmp[string(item.Metric["cluster_id"])+qps.Timestamp.String()] = &clusterInfo{
				OrgID:       string(item.Metric["org_id"]),
				Region:      string(item.Metric["region"]),
				TenantID:    string(item.Metric["tenant_id"]),
				TenantName:  string(item.Metric["tenant_name"]),
				ClusterID:   string(item.Metric["cluster_id"]),
				ClusterName: string(item.Metric["cluster_name"]),
				DeployType:  string(item.Metric["deploy_type"]),
				QPS:         float64(qps.Value),
				Timestamp:   int64(qps.Timestamp) / 1000, // ms to s
			}
		}
	}

	// add duration num to result map
	query = fmt.Sprintf("%s[%dd]", getQPSQueryWithSearch("sum_over_time(clinic_diagnosis_active_qps_duration%s)", threshold, search, region, deployType), timeRangeDays)
	log.Info("active qps duration promQL:" + query)
	result, err = client.PostForm(
		queryURL,
		url.Values{
			"query": {query},
			"time":  {fmt.Sprintf("%d", endTimeInt)},
			"step":  {"1d"}, // only usable for virtorametric
		},
	)
	if err != nil {
		log.Error(err.Error())
		_ = c.Error(error2.ErrInternal.New("internal error"))
	}
	defer result.Body.Close()
	var durationResp metrics.PromQueryResponse
	d = json.NewDecoder(result.Body)
	err = d.Decode(&durationResp)
	if err != nil {
		log.Error("decode active aps query result failed")
		_ = c.Error(error2.ErrInternal.New("internal error"))
		return
	}
	for _, item := range durationResp.Data.Result {
		for _, duration := range item.Values {
			key := string(item.Metric["cluster_id"]) + duration.Timestamp.String()
			if _, ok := tmp[key]; ok {
				tmp[key].Duration = float64(duration.Value)
			}
		}
	}

	// convert map to slice, and sort
	for _, item := range tmp {
		resp.Items = append(resp.Items, item)
	}
	resp.Total = len(resp.Items)
	sort.Slice(resp.Items, func(i, j int) bool {
		return strings.Compare(resp.Items[i].Region, resp.Items[j].Region) < 0
	})

	c.JSON(http.StatusOK, resp)
}

// not used now
func (gd *GlobalDashboardService) GetActiveQPSDetail(c *gin.Context) {
	u := session.GetUser(c)
	if u == nil {
		_ = c.Error(error2.ErrNotFound.New("the user is not found"))
		return
	}

	if !u.Internal {
		_ = c.Error(error2.ErrForbidden.New("permission deny"))
		return
	}

	endTimeStr := c.Query("end_ts")
	if endTimeStr == "" {
		_ = c.Error(error2.ErrBadRequest.New("empty end_ts"))
		return
	}
	endTimeInt, err := strconv.ParseInt(endTimeStr, 10, 64)
	if err != nil {
		_ = c.Error(error2.ErrBadRequest.New("invalid end_ts"))
		return
	}
	endTime := time.Unix(endTimeInt, 0)

	beginTimeStr := c.Query("begin_ts")
	if beginTimeStr == "" {
		_ = c.Error(error2.ErrBadRequest.New("empty begin_ts"))
		return
	}
	beginTimeInt, err := strconv.ParseInt(beginTimeStr, 10, 64)
	if err != nil {
		_ = c.Error(error2.ErrBadRequest.New("invalid begin_ts"))
		return
	}
	timeRange := fmt.Sprintf("%ds", endTimeInt-beginTimeInt)
	threshold := c.Query("threshold")
	if threshold == "" {
		_ = c.Error(error2.ErrBadRequest.New("empty threshold"))
		return
	}

	_ = common.ParsePaginationQuery(c.Request.URL.RequestURI())

	metricConfig := gd.cfg.DiagnosisConfig.MetricConfig.SelectAddr
	// QPS metrics are stored at org 8899
	address := fmt.Sprintf("http://%s:%d/select/8899/prometheus", metricConfig.Host, metricConfig.Port)
	client, err := prometheusC.NewClient(prometheusC.Config{Address: address})
	if err != nil {
		_ = c.Error(error2.ErrInternal.New("failed to create TSDB client"))
		return
	}
	api := prometheusAPI.NewAPI(client)
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	tmp := make(map[string]*clusterInfo)
	resp := ActiveQPSResp{Total: 0, Items: make([]*clusterInfo, 0)}
	querySuffix := getQPSQuerySuffix(
		threshold,
		timeRange,
		c.QueryArray("region"),
		c.QueryArray("cluster_id"),
		c.QueryArray("cluster_name"),
		c.QueryArray("tenant_id"),
		c.QueryArray("tenant_name"),
		c.QueryArray("org_id"),
	)
	log.Info("qps query suffix: " + querySuffix)

	// store result of qps metric to a map
	result, _, _ := api.Query(ctx, "clinic_diagnosis_active_qps_max"+querySuffix, endTime)
	if result == nil {
		// return empty list
		log.Info("qps result is empty")
		c.JSON(http.StatusOK, resp)
		return
	}
	resultM := result.(prometheusModel.Matrix)
	for _, item := range resultM {
		for _, qps := range item.Values {
			tmp[string(item.Metric["cluster_id"])+qps.Timestamp.String()] = &clusterInfo{
				OrgID:       string(item.Metric["org_id"]),
				Region:      string(item.Metric["region"]),
				TenantID:    string(item.Metric["tenant_id"]),
				TenantName:  string(item.Metric["tenant_name"]),
				ClusterID:   string(item.Metric["cluster_id"]),
				ClusterName: string(item.Metric["cluster_name"]),
				DeployType:  string(item.Metric["deploy_type"]),
				QPS:         float64(qps.Value),
				Timestamp:   int64(qps.Timestamp) / 1000, // ms to s
			}
		}
	}

	// add duration num to result map
	result, _, _ = api.Query(ctx, "clinic_diagnosis_active_qps_duration"+querySuffix, endTime)
	if result == nil {
		// return empty list
		log.Info("duration result is empty")
		c.JSON(http.StatusOK, resp)
		return
	}
	resultM = result.(prometheusModel.Matrix)
	for _, item := range resultM {
		for _, duration := range item.Values {
			key := string(item.Metric["cluster_id"]) + duration.Timestamp.String()
			if _, ok := tmp[key]; ok {
				tmp[key].Duration = float64(duration.Value)
			}
		}
	}

	// add linked report id to result map
	result, _, _ = api.Query(ctx, fmt.Sprintf("clinic_diagnosis_active_qps_report[%s]", timeRange), endTime)
	if result != nil {
		resultM = result.(prometheusModel.Matrix)
		for _, item := range resultM {
			for _, reportID := range item.Values {
				key := string(item.Metric["cluster_id"]) + reportID.Timestamp.String()
				if _, ok := tmp[key]; ok {
					uuid, err := model.ReportIDtoUUID(uint(reportID.Value))
					if err == nil {
						tmp[key].ReportUUID = &uuid
					}
				}
			}
		}
	}

	// convert map to slice, and sort
	for _, item := range tmp {
		resp.Items = append(resp.Items, item)
	}
	resp.Total = len(resp.Items)
	sort.Slice(resp.Items, func(i, j int) bool {
		return strings.Compare(resp.Items[i].Region, resp.Items[j].Region) < 0
	})

	c.JSON(http.StatusOK, resp)
}

func getQPSQuerySuffix(threshold, timerange string, regionFilter, clusterIDFilter, clusterNameFilter, tenantIDFilter, tenantNameFilter, orgIDFilter []string) string {
	labels := []string{
		fmt.Sprintf("threshold=\"%s\"", threshold),
	}
	if len(regionFilter) > 0 && regionFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("region=~\"%s\"", strings.Join(regionFilter, "|")))
	}
	if len(clusterIDFilter) > 0 && clusterIDFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("cluster_id=~\"%s\"", strings.Join(clusterIDFilter, "|")))
	}
	if len(clusterNameFilter) > 0 && clusterNameFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("cluster_name=~\"%s\"", strings.Join(clusterNameFilter, "|")))
	}
	if len(tenantIDFilter) > 0 && tenantIDFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("tenant_id=~\"%s\"", strings.Join(tenantIDFilter, "|")))
	}
	if len(tenantNameFilter) > 0 && tenantNameFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("tenant_name=~\"%s\"", strings.Join(tenantNameFilter, "|")))
	}
	if len(orgIDFilter) > 0 && orgIDFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("org_id=~\"%s\"", strings.Join(orgIDFilter, "|")))
	}
	return fmt.Sprintf("{%s}[%s]", strings.Join(labels, ","), timerange)
}

func getQPSQueryWithSearch(mtc, threshold, search string, regionFilter, deployTypeFilter []string) string {
	querys := make([]string, 0)
	labels := []string{
		fmt.Sprintf("threshold=\"%s\"", threshold),
	}
	if len(regionFilter) > 0 && regionFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("region=~\"%s\"", strings.Join(regionFilter, "|")))
	}
	if len(deployTypeFilter) > 0 && deployTypeFilter[0] != "" {
		labels = append(labels, fmt.Sprintf("deploy_type=~\"%s\"", strings.Join(deployTypeFilter, "|")))
	}

	if search != "" {
		labelsWithSearch := append(labels, fmt.Sprintf("cluster_id=~\"(?i).*%s.*\"", search))
		querys = append(querys, fmt.Sprintf(mtc, fmt.Sprintf("{%s}[1d]", strings.Join(labelsWithSearch, ","))))

		labelsWithSearch = append(labels, fmt.Sprintf("cluster_name=~\"(?i).*%s.*\"", search))
		querys = append(querys, fmt.Sprintf(mtc, fmt.Sprintf("{%s}[1d]", strings.Join(labelsWithSearch, ","))))

		labelsWithSearch = append(labels, fmt.Sprintf("tenant_id=~\"(?i).*%s.*\"", search))
		querys = append(querys, fmt.Sprintf(mtc, fmt.Sprintf("{%s}[1d]", strings.Join(labelsWithSearch, ","))))

		labelsWithSearch = append(labels, fmt.Sprintf("tenant_name=~\"(?i).*%s.*\"", search))
		querys = append(querys, fmt.Sprintf(mtc, fmt.Sprintf("{%s}[1d]", strings.Join(labelsWithSearch, ","))))

		return fmt.Sprintf("(%s)", strings.Join(querys, ","))
	} else {
		return fmt.Sprintf(mtc, fmt.Sprintf("{%s}[1d]", strings.Join(labels, ",")))
	}
}