package regionalservice

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/gin-gonic/gin"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/utils/pointer"

	"k8s.io/apimachinery/pkg/api/resource"

	"github.com/tidbcloud/observability/pkg/common/configs"

	prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
	v1 "github.com/tidbcloud/observability/api/regionalserver/resourceworker/v1"
	"github.com/tidbcloud/observability/pkg/common/utils"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	"github.com/stretchr/testify/assert"
	"github.com/tidbcloud/observability/pkg/common/constants"
	"github.com/tidbcloud/observability/pkg/common/errors"
	"github.com/tidbcloud/observability/pkg/common/message"
	"github.com/tidbcloud/observability/pkg/common/structs"
	"github.com/tidbcloud/observability/pkg/resourcemanager"
	"github.com/tidbcloud/observability/test/mockresourcemanager"
	"go.uber.org/mock/gomock"
)

func mockCreateReq() message.CreateClusterReq {
	return message.CreateClusterReq{
		ClusterBaseInfo: structs.ClusterBaseInfo{
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.ProvidedVendor(defaultVendor),
				Region: defaultRegion,
			},
			TenantID:      "t",
			ProjectID:     "p",
			ApplicationID: "a",
			BizType:       "b",
			ShareMode:     constants.EXCLUSIVE,
		},
		MonitorSpecID:  "default",
		AgentAccessARN: "arn:aws:iam::123456789012:root",
	}
}

func mockPrometheusRuleSpec(name string) prometheusv1.PrometheusRuleSpec {
	return prometheusv1.PrometheusRuleSpec{
		Groups: []prometheusv1.RuleGroup{
			0: {
				Name:     name,
				Interval: "5",
				Rules: []prometheusv1.Rule{
					{
						Alert: "HighRequestLatency",
						Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.5"),
						For:   "10m",
						Labels: map[string]string{
							"severity": "critical",
						},
						Annotations: map[string]string{
							"summary": "High request latency",
						},
					},
				},
			},
		},
	}
}
func mockPrometheusRule(id string, tenantID, projectID string) *prometheusv1.PrometheusRule {
	return &prometheusv1.PrometheusRule{
		ObjectMeta: metav1.ObjectMeta{
			Name:      id,
			Namespace: "default-" + tenantID,
			Labels: map[string]string{
				constants.LabelClusterID:     id,
				constants.LabelVendor:        "aws",
				constants.LabelRegion:        "us-east-2",
				constants.LabelTenantID:      tenantID,
				constants.LabelProjectID:     projectID,
				constants.LabelApplicationID: "default",
				constants.LabelBizType:       "default",
			},
			Annotations: map[string]string{
				constants.AnnotationNote: "note111",
			},
		},
		Spec: prometheusv1.PrometheusRuleSpec{
			Groups: []prometheusv1.RuleGroup{
				0: {
					Name:     "example",
					Interval: "5",
					Rules: []prometheusv1.Rule{
						{
							Alert: "HighRequestLatency",
							Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.5"),
							For:   "10m",
							Labels: map[string]string{
								"severity":  "critical",
								"alerttype": "customer",
							},
							Annotations: map[string]string{
								"summary": "High request latency",
							},
						},
					},
				},
			},
		},
	}
}

func Test_CreateCluster(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	dataAccountRW := mockresourcemanager.NewMockDataAccountReaderWriter(ctrl)
	resourcemanager.MockDataAccountReaderWriter(dataAccountRW)

	namespaceRW := mockresourcemanager.NewMockNamespaceReaderWriter(ctrl)
	resourcemanager.MockNamespaceReaderWriter(namespaceRW)
	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	resourcemanager.MockVMRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).AnyTimes()
	promRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes()
	promRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	req := mockCreateReq()
	namespaceMock := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: NamespaceName(req.BizType, req.TenantID),
			Labels: map[string]string{
				constants.LabelBizType:  req.BizType,
				constants.LabelTenantID: req.TenantID,
			},
		},
	}
	t.Run("check id failed", func(t *testing.T) {
		clusterRW.EXPECT().IsClusterIDExists(gomock.Any(), gomock.Any()).Return(true, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				ID: "id",
				CommonReqInfo: structs.CommonReqInfo{
					Vendor: constants.ProvidedVendor(defaultVendor),
					Region: defaultRegion,
				},
			},
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})
	t.Run("duplicated cluster id", func(t *testing.T) {
		clusterRW.EXPECT().IsClusterIDExists(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				ID: "id",
				CommonReqInfo: structs.CommonReqInfo{
					Vendor: constants.ProvidedVendor(defaultVendor),
					Region: defaultRegion,
				},
			},
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())
	})
	t.Run("duplicated cluster", func(t *testing.T) {
		returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), req, "", nil)
		clusters := &v1.ClusterList{
			Items: []v1.Cluster{*returnCluster},
		}
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(clusters, nil).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
		assert.Equal(t, errors.DuplicatedCluster, err.(errors.Error).Code())
	})
	t.Run("check duplicated cluster failed", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})
	t.Run("prepare namespace failed", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(nil, nil).Times(1)
		namespaceRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
	})
	t.Run("prepare bucket failed", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		objectStorageRW.EXPECT().CreateObjectStorage(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
	})
	t.Run("prepare iam failed", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
	})
	t.Run("wrong spec", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		namespaceRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		//promRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		original := req.MonitorSpecID
		req.MonitorSpecID = "wrong"
		defer func() {
			req.MonitorSpecID = original
		}()
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
	})
	t.Run("create failed", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		namespaceRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().CreateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		//promRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		//promRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.Error(t, err)
	})

	t.Run("normal", func(t *testing.T) {
		originalSpec := req.MonitorSpecID
		req.MonitorSpecID = ""
		defer func() {
			req.MonitorSpecID = originalSpec
		}()
		original := req.AgentTemplateName
		req.AgentTemplateName = "test"
		defer func() {
			req.AgentTemplateName = original
		}()
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		namespaceRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().CreateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.NoError(t, err)
	})
	t.Run("normal gcp with role existed", func(t *testing.T) {
		originalSpec := req.MonitorSpecID
		req.MonitorSpecID = ""
		original := req.AgentTemplateName
		req.AgentTemplateName = "test"
		originalConf := configs.GetServiceInfo()
		configs.Init(structs.ServiceInfo{
			Vendor: string(constants.GCP),
			Region: "us-west1",
			Env:    constants.EnvDev,
			AZConfig: map[string]structs.AvailabilityZoneConfig{
				"zone1": {
					EndpointServiceID: "e1",
					IngressClassName:  "i1",
				},
			},
			AccountInfo: structs.AccountInfoConfig{
				Oidc:    "gcp-autonomous-dev.svc.id.goog",
				Account: "gcp-autonomous-dev.iam.gserviceaccount.com",
				Region:  "us-west1",
			},
			MonitorSpec:    originalConf.MonitorSpec,
			BizTypeConfigs: originalConf.BizTypeConfigs,
		})
		defer func() {
			configs.Init(originalConf)
			req.MonitorSpecID = originalSpec
			req.AgentTemplateName = original
		}()
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Gcp: v1.GCPIAM{
					AssumeRoleAccounts: []string{"test"},
				},
			},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().CreateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		req.AgentAccessARN = "regional-server@gcp-autonomous-prod.iam.gserviceaccount.com"
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.NoError(t, err)
	})
	t.Run("normal azure", func(t *testing.T) {
		originalReq := req
		req.ShareMode = constants.SHARED
		req.MonitorSpecID = ""
		req.AgentTemplateName = "test"
		originalConf := configs.GetServiceInfo()
		configs.Init(structs.ServiceInfo{
			Vendor: string(constants.Azure),
			Region: "eastus",
			Env:    constants.EnvDev,
			AZConfig: map[string]structs.AvailabilityZoneConfig{
				"zone1": {
					EndpointServiceID: "e1",
					IngressClassName:  "i1",
					DefaultDataAZ:     true,
				},
			},
			AccountInfo: structs.AccountInfoConfig{
				Region:                   "eastus",
				SubscriptionID:           "test",
				StorageAccount:           "test",
				BlobStorageResourceGroup: "test",
			},
			VMClusterInfo: []structs.VictoriaMetricsCluster{
				{
					VMClusterName: "test",
				},
			},
			MonitorSpec:    originalConf.MonitorSpec,
			BizTypeConfigs: originalConf.BizTypeConfigs,
		})
		defer func() {
			configs.Init(originalConf)
			req = originalReq
		}()
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		dataAccountRW.EXPECT().UniqueInt32ForCluster(gomock.Any()).Return(int32(111), nil).AnyTimes()
		namespaceRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().CreateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		_, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.NoError(t, err)
	})
}

func Test_PrepareBucketResource(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	req := mockCreateReq()
	t.Run("existed", func(t *testing.T) {
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		//objectStorageRW.EXPECT().CreateObjectStorage(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		err := prepareBucketResource(context.TODO(), req)
		assert.NoError(t, err)
	})
	t.Run("create failed", func(t *testing.T) {
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		objectStorageRW.EXPECT().CreateObjectStorage(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		err := prepareBucketResource(context.TODO(), req)
		assert.Error(t, err)
	})
}

func Test_prepareIAMResource(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	req := mockCreateReq()
	origin := req.Vendor
	req.Vendor = constants.GCP
	defer func() {
		req.Vendor = origin
	}()
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	configs.Init(structs.ServiceInfo{
		ServerName: constants.ObservabilityRegionalServer,
		Port:       8080,
		Vendor:     string(constants.GCP),
		Region:     defaultRegion,
		Host:       "127.0.0.1",
		Env:        constants.EnvDev,
		MonitorSpec: map[string]structs.MonitorSpec{
			"default": {
				Cpu:            "10m",
				Memory:         "200Mi",
				InitialStorage: "40Gi",
				OverSell:       1,
				IsDefault:      true,
			},
		},
		AZConfig: map[string]structs.AvailabilityZoneConfig{
			"zone1": {
				EndpointServiceID: "e1",
				IngressClassName:  "i1",
			},
			"zone2": {
				EndpointServiceID: "e2",
				IngressClassName:  "i2",
			},
		},
		AccountInfo: structs.AccountInfoConfig{
			Oidc:    "oidc.eks.us-west-2.amazonaws.com/id/005D92AA81DE2DCE14A2991BAC54FEF8",
			OidcARN: "arn:aws:iam::552185537300:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/005D92AA81DE2DCE14A2991BAC54FEF8",
			Account: "aaaa",
			Region:  "us-west-2",
		},
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			"b": {
				AgentManagerARN: "aaa",
			},
			"tidbcloud": {
				AgentManagerARN: "role2",
			},
		},
	})

	t.Run("isExisted error", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, errors.New(errors.UnrecognizedError, "")).Times(1)
		req1 := req
		req1.AgentAccessARN = "test@gcp-autonomous-dev.iam.gserviceaccount.com"
		err := prepareIAMResource(context.TODO(), req1)
		assert.Error(t, err)
	})
	t.Run("existed and get role error", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{}, errors.New(errors.UnrecognizedError, "")).Times(1)
		req1 := req
		req1.AgentAccessARN = "test@gcp-autonomous-dev.iam.gserviceaccount.com"
		err := prepareIAMResource(context.TODO(), req1)
		assert.Error(t, err)
	})
	t.Run("existed and need update", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: buildIAMObjectMeta(req),
			Spec: v1.IAMSpec{Gcp: v1.GCPIAM{
				AssumeRoleAccounts: []string{
					NamespaceName(req.BizType, req.TenantID),
					"existing@gcp-autonomous-dev.iam.gserviceaccount.com",
				},
			}},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		req1 := req
		req1.AgentAccessARN = "test1@gcp-autonomous-dev.iam.gserviceaccount.com"
		err := prepareIAMResource(context.TODO(), req1)
		assert.NoError(t, err)
	})
	t.Run("does not exist and createRole error", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		req1 := req
		req1.AgentAccessARN = "test@gcp-autonomous-dev.iam.gserviceaccount.com"
		err := prepareIAMResource(context.TODO(), req1)
		assert.Error(t, err)
	})
	t.Run("duplicated RoleName", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(errors.New(errors.DuplicatedRoleName, "")).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: buildIAMObjectMeta(req),
			Spec: v1.IAMSpec{Gcp: v1.GCPIAM{
				AssumeRoleAccounts: []string{
					NamespaceName(req.BizType, req.TenantID),
					"existing@gcp-autonomous-dev.iam.gserviceaccount.com",
				},
			}},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		req1 := req
		req1.AgentAccessARN = "test2@gcp-autonomous-dev.iam.gserviceaccount.com"
		err := prepareIAMResource(context.TODO(), req1)
		assert.NoError(t, err)
	})
	t.Run("does not exist and createRole", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		nsRW := mockresourcemanager.NewMockNamespaceReaderWriter(ctrl)
		resourcemanager.MockNamespaceReaderWriter(nsRW)
		nsRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		req1 := req
		req1.AgentAccessARN = "test@gcp-autonomous-dev.iam.gserviceaccount.com"
		err := prepareIAMResource(context.TODO(), req1)
		assert.NoError(t, err)
	})
	t.Run("wrong arn", func(t *testing.T) {
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		req1 := req
		req1.AgentAccessARN = "test"
		err := prepareIAMResource(context.TODO(), req1)
		assert.Error(t, err)
	})

}

func Test_prepareIAMResourceAzureAgentRoleNotNeeded(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	configs.Init(structs.ServiceInfo{
		Vendor: string(constants.Azure),
		Region: defaultRegion,
		Env:    constants.EnvDev,
		AccountInfo: structs.AccountInfoConfig{
			SubscriptionID:           "sub-id",
			BlobStorageResourceGroup: "storage-rg",
			StorageAccount:           "storage-account",
		},
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			"tidbcloud": {},
		},
	})

	req := mockCreateReq()
	req.Vendor = constants.Azure
	req.BizType = "tidbcloud"
	req.ShareMode = constants.SHARED
	req.ApplicationID = "app-1"
	req.AgentAccessARN = "principal-1"

	t.Run("already contains access arn", func(t *testing.T) {
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Azure: &v1.AzureIAM{AdditionalRoleMembers: []string{"principal-1"}},
			},
		}, nil).Times(1)

		err := prepareIAMResource(context.TODO(), req)
		assert.NoError(t, err)
	})

	t.Run("append access arn when missing", func(t *testing.T) {
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Azure: &v1.AzureIAM{AdditionalRoleMembers: []string{"principal-2"}},
			},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		err := prepareIAMResource(context.TODO(), req)
		assert.NoError(t, err)
	})

	t.Run("create iam when role not found", func(t *testing.T) {
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.RoleNotFound, "")).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		err := prepareIAMResource(context.TODO(), req)
		assert.NoError(t, err)
	})
}

func Test_prepareIAMResourceGCPAgentRoleNotNeeded(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	configs.Init(structs.ServiceInfo{
		Vendor: string(constants.GCP),
		Region: defaultRegion,
		Env:    constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			"datamigration-shared": {},
		},
	})

	req := mockCreateReq()
	req.Vendor = constants.GCP
	req.BizType = "datamigration-shared"
	req.ShareMode = constants.SHARED
	req.ApplicationID = "app-1"
	req.AgentAccessARN = "principal@gcp-project.iam.gserviceaccount.com"

	t.Run("already contains access arn", func(t *testing.T) {
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Gcp: v1.GCPIAM{AssumeRoleAccounts: []string{"serviceAccount:principal@gcp-project.iam.gserviceaccount.com"}},
			},
		}, nil).Times(1)

		err := prepareIAMResource(context.TODO(), req)
		assert.NoError(t, err)
	})

	t.Run("append access arn when missing", func(t *testing.T) {
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Gcp: v1.GCPIAM{AssumeRoleAccounts: []string{"serviceAccount:other@gcp-project.iam.gserviceaccount.com"}},
			},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		err := prepareIAMResource(context.TODO(), req)
		assert.NoError(t, err)
	})

	t.Run("create iam when role not found", func(t *testing.T) {
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.RoleNotFound, "")).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		err := prepareIAMResource(context.TODO(), req)
		assert.NoError(t, err)
	})
}

func Test_DeleteCluster(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)
	archivedRW := mockresourcemanager.NewMockArchivedClusterReaderWriter(ctrl)
	resourcemanager.MockArchivedClusterReaderWriter(archivedRW)
	archivedRW.EXPECT().Archive(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRW.EXPECT().GetCluster(gomock.Any(), "not found").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()

	returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	clusterRW.EXPECT().GetCluster(gomock.Any(), "validID").Return(returnCluster, nil).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "empty").Return(nil, nil).AnyTimes()

	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	resourcemanager.MockVMRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)
	iamRW.EXPECT().DeleteRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	deleting, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	deleting.Status.Phase = constants.DELETING

	clusterRW.EXPECT().GetCluster(gomock.Any(), "deleting").Return(deleting, nil).AnyTimes()

	paused, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	paused.Status.Phase = constants.PAUSED
	clusterRW.EXPECT().GetCluster(gomock.Any(), "paused").Return(paused, nil).AnyTimes()

	clusterRW.EXPECT().GetCluster(gomock.Any(), "parentID").Return(&v1.Cluster{
		Spec: v1.ClusterSpec{
			VictoriaAccount: v1.VictoriaAccount{
				AccountID: 999,
				ProjectID: 0,
			},
		},
	}, nil).AnyTimes()

	t.Run("id not found", func(t *testing.T) {
		_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "not found",
		})
		assert.Error(t, err)
	})
	t.Run("being deleted", func(t *testing.T) {
		_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "deleting",
		})
		assert.NoError(t, err)
	})
	t.Run("delete failed", func(t *testing.T) {
		clusterRW.EXPECT().SoftDeleteCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "validID",
		})
		assert.Error(t, err)
	})
	t.Run("parse baseInfo failed", func(t *testing.T) {
		_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "empty",
		})
		assert.Error(t, err)
	})
	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().SoftDeleteCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		resp, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "validID",
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, resp.Vendor)
		assert.NotEmpty(t, resp.Region)
		assert.NotEmpty(t, resp.TenantID)
		assert.NotEmpty(t, resp.ProjectID)
		assert.NotEmpty(t, resp.ApplicationID)
		assert.NotEmpty(t, resp.BizType)
	})
	t.Run("paused error", func(t *testing.T) {
		clusterRW.EXPECT().DeleteCluster(context.TODO(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "paused",
		})
		assert.Error(t, err)
	})
	t.Run("paused ok", func(t *testing.T) {
		clusterRW.EXPECT().DeleteCluster(context.TODO(), gomock.Any()).Return(nil).Times(1)

		_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "paused",
		})
		assert.NoError(t, err)
	})
	t.Run("delete iam role in azure", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			Vendor: string(constants.Azure),
			Region: "eastus",
			Env:    constants.EnvDev,
			AZConfig: map[string]structs.AvailabilityZoneConfig{
				"zone1": {
					EndpointServiceID: "e1",
					IngressClassName:  "i1",
					DefaultDataAZ:     true,
				},
			},
			AccountInfo: structs.AccountInfoConfig{
				Region:                   "eastus",
				SubscriptionID:           "test",
				StorageAccount:           "test",
				BlobStorageResourceGroup: "test",
			},
			VMClusterInfo: []structs.VictoriaMetricsCluster{
				{
					VMClusterName: "test",
				},
			},
		})
		clusterRW.EXPECT().DeleteCluster(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Azure: &v1.AzureIAM{
					WorkloadServiceAccounts: []string{"tidba:vector"},
				},
			},
		}, nil).Times(3)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		sharedCluster, err := BuildClusterResourceRequirements(context.TODO(), mockSharedReq(withAgentAzure()), "", nil)
		assert.NoError(t, err)
		sharedCluster.Name = "sharedAzure"
		clusterRW.EXPECT().GetCluster(gomock.Any(), "sharedAzure").Return(sharedCluster, nil).Times(2)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{
			Items: []v1.Cluster{
				*sharedCluster,
				{
					ObjectMeta: metav1.ObjectMeta{
						Name:   "test2",
						Labels: returnCluster.Labels,
					},
				},
			},
		}, nil).Times(1)
		_, err = GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "sharedAzure",
		})
		assert.NoError(t, err)

		t.Log("delete azure cluster roles' workloadSA")
		sharedCopy := sharedCluster.DeepCopy()
		sharedCopy.Name = "test111"
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{
			Items: []v1.Cluster{
				*sharedCluster,
				*sharedCopy,
			},
		}, nil).Times(1)
		_, err = GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "sharedAzure",
		})
		assert.NoError(t, err)
	})

	t.Run("delete aws dataplane iam policies -- update dataplane role", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			Vendor: string(constants.AWS),
			Region: "us-west-2",
			Env:    constants.EnvDev,
			AZConfig: map[string]structs.AvailabilityZoneConfig{
				"zone1": {
					EndpointServiceID: "e1",
					IngressClassName:  "i1",
					DefaultDataAZ:     true,
				},
			},
			AccountInfo: structs.AccountInfoConfig{
				Region: "us-west-2",
			},
			VMClusterInfo: []structs.VictoriaMetricsCluster{
				{
					VMClusterName: "test",
				},
			},
		})
		clusterRW.EXPECT().DeleteCluster(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

		iamRW.EXPECT().IsRoleExists(gomock.Any(), "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2", gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2", gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2-account111", gomock.Any()).Return(true, nil).Times(1)

		policy := AssumeRolePolicy{
			Version: "2012-10-17",
			Statement: []Statement{
				{
					Effect: "Allow",
					Principal: map[string]interface{}{
						TrustPolicyPrincipalKeyFederated: "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/TEST1",
					},
				},
				{
					Effect: "Allow",
					Principal: map[string]interface{}{
						TrustPolicyPrincipalKeyFederated: "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/OTHER123456789",
					},
				},
			},
		}
		policyJSON, _ := json.Marshal(policy)
		iamRW.EXPECT().GetRole(gomock.Any(), "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2-account111", gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Aws: v1.AWSIAM{
					AssumeRolePolicyDocument: base64.StdEncoding.EncodeToString(policyJSON),
				},
			},
		}, nil).Times(2)

		o11yPolicy := AssumeRolePolicy{
			Version: "2012-10-17",
			Statement: []Statement{
				{
					Effect: "Allow",
					Principal: map[string]interface{}{
						TrustPolicyPrincipalKeyAWS: "arn:aws:iam::123456789012:role/o11yAD-mG88wrcjPdGFE9o5etFzkJ",
					},
				},
			},
		}
		o11yPolicyJSON, _ := json.Marshal(o11yPolicy)
		iamRW.EXPECT().GetRole(gomock.Any(), "o11y-agent-dev-tidbcloud-tenant222-us-west-2", gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Aws: v1.AWSIAM{
					AssumeRolePolicyDocument: base64.StdEncoding.EncodeToString(o11yPolicyJSON),
				},
			},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil)

		awsCluster, err := BuildClusterResourceRequirements(context.TODO(), mockSharedReq(withAgentAWS()), "", nil)
		assert.NoError(t, err)
		awsCluster.Name = "aws-cluster"

		otherCluster := awsCluster.DeepCopy()
		otherCluster.Name = "other-cluster"
		otherCluster.Spec.Agent.AWS.OIDC = "oidc.eks.us-west-2.amazonaws.com/id/OTHER123456789"

		clusterRW.EXPECT().GetCluster(gomock.Any(), "aws-cluster").Return(awsCluster, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{
			Items: []v1.Cluster{*awsCluster, *otherCluster},
		}, nil).Times(1)
		clusterRW.EXPECT().ListClusters(gomock.Any(), gomock.Any()).Return(&v1.ClusterList{
			Items: []v1.Cluster{*awsCluster, *otherCluster},
		}, nil).Times(1)

		_, err = GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "aws-cluster",
		})
		assert.NoError(t, err)
	})

	t.Run("delete aws dataplane iam policies -- delete dataplane role & update o11y role", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			Vendor: string(constants.AWS),
			Region: "us-west-2",
			Env:    constants.EnvDev,
			AZConfig: map[string]structs.AvailabilityZoneConfig{
				"zone1": {
					EndpointServiceID: "e1",
					IngressClassName:  "i1",
					DefaultDataAZ:     true,
				},
			},
			AccountInfo: structs.AccountInfoConfig{
				Region: "us-west-2",
			},
			VMClusterInfo: []structs.VictoriaMetricsCluster{
				{
					VMClusterName: "test",
				},
			},
		})
		clusterRW.EXPECT().DeleteCluster(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
		iamRW.EXPECT().IsRoleExists(gomock.Any(), "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2", gomock.Any()).Return(false, nil).Times(1)
		policy := AssumeRolePolicy{
			Version: "2012-10-17",
			Statement: []Statement{
				{
					Effect: "Allow",
					Principal: map[string]interface{}{
						TrustPolicyPrincipalKeyFederated: "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/TEST1",
					},
				},
			},
		}
		policyJSON, _ := json.Marshal(policy)
		dataplaneIAM := &v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Name: "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2-account111",
				Labels: map[string]string{
					constants.LabelAlias: "o11yAD-mG88wrcjPdGFE9o5etFzkJ",
				},
			},
			Spec: v1.IAMSpec{
				Aws: v1.AWSIAM{
					AssumeRolePolicyDocument: base64.StdEncoding.EncodeToString(policyJSON),
				},
			},
		}
		iamRW.EXPECT().GetRole(gomock.Any(), "o11y-agent-data-dev-tidbcloud-tenant222-us-west-2-account111", gomock.Any()).Return(dataplaneIAM, nil).Times(1)

		o11yPolicy := AssumeRolePolicy{
			Version: "2012-10-17",
			Statement: []Statement{
				{
					Effect: "Allow",
					Principal: map[string]interface{}{
						TrustPolicyPrincipalKeyFederated: "arn:aws:iam::123456789012:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/O11y",
					},
				},
				{
					Effect: "Allow",
					Principal: map[string]interface{}{
						TrustPolicyPrincipalKeyAWS: "arn:aws:iam::123456789012:role/o11yAD-mG88wrcjPdGFE9o5etFzkJ",
					},
				},
			},
		}
		o11yPolicyJSON, _ := json.Marshal(o11yPolicy)
		iamRW.EXPECT().GetRole(gomock.Any(), "o11y-agent-dev-tidbcloud-tenant222-us-west-2", gomock.Any()).Return(&v1.IAM{
			Spec: v1.IAMSpec{
				Aws: v1.AWSIAM{
					AssumeRolePolicyDocument: base64.StdEncoding.EncodeToString(o11yPolicyJSON),
				},
			},
		}, nil).Times(2)

		awsCluster, err := BuildClusterResourceRequirements(context.TODO(), mockSharedReq(withAgentAWS()), "", nil)
		assert.NoError(t, err)
		awsCluster.Name = "aws-cluster"

		clusterRW.EXPECT().GetCluster(gomock.Any(), "aws-cluster").Return(awsCluster, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{
			Items: []v1.Cluster{*awsCluster},
		}, nil).Times(1)
		iamRW.EXPECT().GetRoles(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAMList{
			Items: []v1.IAM{*dataplaneIAM},
		}, nil).Times(1)
		iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		clusterRW.EXPECT().ListClusters(gomock.Any(), gomock.Any()).Return(&v1.ClusterList{
			Items: []v1.Cluster{*awsCluster},
		}, nil).Times(1)

		_, err = GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
			ID: "aws-cluster",
		})
		assert.NoError(t, err)
	})
}

func Test_DeleteClusterAzureRemoveAccessARN(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	resourcemanager.MockVMRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	original := configs.GetServiceInfo()
	defer configs.Init(original)
	configs.Init(structs.ServiceInfo{Vendor: string(constants.Azure), Region: "eastus", Env: constants.EnvDev})

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "c1",
			Namespace: NamespaceName("tidbcloud", "tenant1"),
			Labels: map[string]string{
				constants.LabelVendor:        string(constants.Azure),
				constants.LabelRegion:        "eastus",
				constants.LabelBizType:       "tidbcloud",
				constants.LabelTenantID:      "tenant1",
				constants.LabelProjectID:     "project1",
				constants.LabelApplicationID: "app1",
				constants.LabelShareMode:     string(constants.EXCLUSIVE),
			},
		},
		Status: v1.ClusterStatus{Phase: constants.ACTIVE},
	}

	clusterRW.EXPECT().GetCluster(gomock.Any(), "c1").Return(cluster, nil).Times(1)
	clusterRW.EXPECT().SoftDeleteCluster(gomock.Any(), "c1").Return(nil).Times(1)

	iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
		Spec: v1.IAMSpec{
			Azure: &v1.AzureIAM{
				AdditionalRoleMembers: []string{"principal-a"},
			},
		},
	}, nil).Times(1)
	iamRW.EXPECT().DeleteRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
		ID:             "c1",
		AgentAccessARN: "principal-a",
	})
	assert.NoError(t, err)
}

func Test_DeleteClusterAzureRemoveAccessARNUpdateRole(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	resourcemanager.MockVMRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	original := configs.GetServiceInfo()
	defer configs.Init(original)
	configs.Init(structs.ServiceInfo{Vendor: string(constants.Azure), Region: "eastus", Env: constants.EnvDev})

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "c1",
			Namespace: NamespaceName("tidbcloud", "tenant1"),
			Labels: map[string]string{
				constants.LabelVendor:        string(constants.Azure),
				constants.LabelRegion:        "eastus",
				constants.LabelBizType:       "tidbcloud",
				constants.LabelTenantID:      "tenant1",
				constants.LabelProjectID:     "project1",
				constants.LabelApplicationID: "app1",
				constants.LabelShareMode:     string(constants.EXCLUSIVE),
			},
		},
		Status: v1.ClusterStatus{Phase: constants.ACTIVE},
	}

	clusterRW.EXPECT().GetCluster(gomock.Any(), "c1").Return(cluster, nil).Times(1)
	clusterRW.EXPECT().SoftDeleteCluster(gomock.Any(), "c1").Return(nil).Times(1)

	target := "principal-a"
	keep := "principal-b"
	iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
		Spec: v1.IAMSpec{
			Azure: &v1.AzureIAM{
				AdditionalRoleMembers:     []string{target, keep},
				LastAdditionalRoleMembers: []string{target, keep},
			},
		},
	}, nil).Times(1)
	iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, iam *v1.IAM) error {
		assert.Equal(t, []string{keep}, iam.Spec.Azure.AdditionalRoleMembers)
		assert.Equal(t, []string{keep}, iam.Spec.Azure.LastAdditionalRoleMembers)
		return nil
	}).Times(1)

	_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
		ID:             "c1",
		AgentAccessARN: target,
	})
	assert.NoError(t, err)
}

func Test_DeleteClusterGCPRemoveAccessARN(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	resourcemanager.MockVMRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	original := configs.GetServiceInfo()
	defer configs.Init(original)
	configs.Init(structs.ServiceInfo{Vendor: string(constants.GCP), Region: defaultRegion, Env: constants.EnvDev})

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "c1",
			Namespace: NamespaceName("tidbcloud", "tenant1"),
			Labels: map[string]string{
				constants.LabelVendor:        string(constants.GCP),
				constants.LabelRegion:        defaultRegion,
				constants.LabelBizType:       "tidbcloud",
				constants.LabelTenantID:      "tenant1",
				constants.LabelProjectID:     "project1",
				constants.LabelApplicationID: "app1",
				constants.LabelShareMode:     string(constants.EXCLUSIVE),
			},
		},
		Status: v1.ClusterStatus{Phase: constants.ACTIVE},
	}

	clusterRW.EXPECT().GetCluster(gomock.Any(), "c1").Return(cluster, nil).Times(1)
	clusterRW.EXPECT().SoftDeleteCluster(gomock.Any(), "c1").Return(nil).Times(1)

	iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
		Spec: v1.IAMSpec{
			Gcp: v1.GCPIAM{
				AssumeRoleAccounts: []string{"serviceAccount:principal-a@gcp-project.iam.gserviceaccount.com"},
			},
		},
	}, nil).Times(1)
	iamRW.EXPECT().DeleteRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)

	_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
		ID:             "c1",
		AgentAccessARN: "principal-a@gcp-project.iam.gserviceaccount.com",
	})
	assert.NoError(t, err)
}

func Test_DeleteClusterGCPRemoveAccessARNUpdateRole(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	resourcemanager.MockVMRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Delete(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	original := configs.GetServiceInfo()
	defer configs.Init(original)
	configs.Init(structs.ServiceInfo{Vendor: string(constants.GCP), Region: defaultRegion, Env: constants.EnvDev})

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "c1",
			Namespace: NamespaceName("tidbcloud", "tenant1"),
			Labels: map[string]string{
				constants.LabelVendor:        string(constants.GCP),
				constants.LabelRegion:        defaultRegion,
				constants.LabelBizType:       "tidbcloud",
				constants.LabelTenantID:      "tenant1",
				constants.LabelProjectID:     "project1",
				constants.LabelApplicationID: "app1",
				constants.LabelShareMode:     string(constants.EXCLUSIVE),
			},
		},
		Status: v1.ClusterStatus{Phase: constants.ACTIVE},
	}

	clusterRW.EXPECT().GetCluster(gomock.Any(), "c1").Return(cluster, nil).Times(1)
	clusterRW.EXPECT().SoftDeleteCluster(gomock.Any(), "c1").Return(nil).Times(1)

	target := "serviceAccount:principal-a@gcp-project.iam.gserviceaccount.com"
	keep := "serviceAccount:principal-b@gcp-project.iam.gserviceaccount.com"
	iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
		Spec: v1.IAMSpec{
			Gcp: v1.GCPIAM{
				AssumeRoleAccounts:     []string{target, keep},
				LastAssumeRoleAccounts: []string{target, keep},
			},
		},
	}, nil).Times(1)
	iamRW.EXPECT().UpdateRole(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, iam *v1.IAM) error {
		assert.Equal(t, []string{keep}, iam.Spec.Gcp.AssumeRoleAccounts)
		assert.Equal(t, []string{keep}, iam.Spec.Gcp.LastAssumeRoleAccounts)
		return nil
	}).Times(1)

	_, err := GetHandler().DeleteCluster(context.TODO(), message.DeleteClusterReq{
		ID:             "c1",
		AgentAccessARN: "principal-a@gcp-project.iam.gserviceaccount.com",
	})
	assert.NoError(t, err)
}

func Test_PauseCluster(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	clusterRW.EXPECT().GetCluster(gomock.Any(), "not found").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "empty").Return(&v1.Cluster{}, nil).AnyTimes()

	returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	returnCluster.Status.Phase = constants.ACTIVE
	clusterRW.EXPECT().GetCluster(gomock.Any(), "validID").Return(returnCluster, nil).AnyTimes()

	pausing, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	pausing.Status.Phase = constants.PAUSED
	clusterRW.EXPECT().GetCluster(gomock.Any(), "pausing").Return(pausing, nil).AnyTimes()

	clusterCanNotBePaused, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	clusterCanNotBePaused.Status.Phase = constants.CREATING
	clusterRW.EXPECT().GetCluster(gomock.Any(), "can not be paused").Return(clusterCanNotBePaused, nil).AnyTimes()

	shared, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	shared.Annotations[constants.AnnotationShareMode] = string(constants.SHARED)
	clusterRW.EXPECT().GetCluster(gomock.Any(), "shared").Return(shared, nil).AnyTimes()

	t.Run("id not found", func(t *testing.T) {
		_, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "not found",
		})
		assert.Error(t, err)
	})
	t.Run("can not be paused", func(t *testing.T) {
		_, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "can not be paused",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterStatusConflict, err.(errors.Error).Code())
	})
	t.Run("being paused", func(t *testing.T) {
		_, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "pausing",
		})
		assert.NoError(t, err)
	})
	t.Run("pause failed", func(t *testing.T) {
		clusterRW.EXPECT().SoftPauseCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "validID",
		})
		assert.Error(t, err)
	})
	t.Run("parse baseInfo failed", func(t *testing.T) {
		_, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "empty",
		})
		assert.Error(t, err)
	})
	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().SoftPauseCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		resp, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "validID",
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, resp.Vendor)
		assert.NotEmpty(t, resp.Region)
		assert.NotEmpty(t, resp.TenantID)
		assert.NotEmpty(t, resp.ProjectID)
		assert.NotEmpty(t, resp.ApplicationID)
		assert.NotEmpty(t, resp.BizType)
	})
	t.Run("share-mode", func(t *testing.T) {
		clusterRW.EXPECT().PauseCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().PauseCluster(context.TODO(), message.PauseClusterReq{
			ID: "shared",
		})
		assert.NoError(t, err)
	})
}

func Test_ResumeCluster(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	clusterRW.EXPECT().GetCluster(gomock.Any(), "not found").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "empty").Return(&v1.Cluster{}, nil).AnyTimes()

	returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	returnCluster.Status.Phase = constants.TO_BE_PAUSED
	clusterRW.EXPECT().GetCluster(gomock.Any(), "validID").Return(returnCluster, nil).AnyTimes()

	resuming, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	resuming.Status.Phase = constants.RESUMING
	clusterRW.EXPECT().GetCluster(gomock.Any(), "resuming").Return(resuming, nil).AnyTimes()

	clusterCanNotBeResumed, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	clusterCanNotBeResumed.Status.Phase = constants.CREATING
	clusterRW.EXPECT().GetCluster(gomock.Any(), "can not be resumed").Return(clusterCanNotBeResumed, nil).AnyTimes()

	shared, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	shared.Annotations[constants.AnnotationShareMode] = string(constants.SHARED)
	clusterRW.EXPECT().GetCluster(gomock.Any(), "shared").Return(shared, nil).AnyTimes()

	t.Run("id not found", func(t *testing.T) {
		_, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "not found",
		})
		assert.Error(t, err)
	})
	t.Run("can not be resumed", func(t *testing.T) {
		_, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "can not be resumed",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterStatusConflict, err.(errors.Error).Code())
	})
	t.Run("resuming", func(t *testing.T) {
		_, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "resuming",
		})
		assert.NoError(t, err)
	})
	t.Run("resume failed", func(t *testing.T) {
		clusterRW.EXPECT().ResumeCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "validID",
		})
		assert.Error(t, err)
	})
	t.Run("parse baseInfo failed", func(t *testing.T) {
		_, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "empty",
		})
		assert.Error(t, err)
	})
	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().ResumeCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		resp, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "validID",
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, resp.Vendor)
		assert.NotEmpty(t, resp.Region)
		assert.NotEmpty(t, resp.TenantID)
		assert.NotEmpty(t, resp.ProjectID)
		assert.NotEmpty(t, resp.ApplicationID)
		assert.NotEmpty(t, resp.BizType)
	})
	t.Run("share-mode", func(t *testing.T) {
		_, err := GetHandler().ResumeCluster(context.TODO(), message.ResumeClusterReq{
			ID: "shared",
		})
		assert.NoError(t, err)
	})

}

func Test_UpdateCluster(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	clusterRW.EXPECT().GetCluster(gomock.Any(), "not found").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()

	returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	returnCluster.Status.Phase = constants.ACTIVE
	clusterRW.EXPECT().GetCluster(gomock.Any(), "validID").Return(returnCluster, nil).AnyTimes()

	clusterUpdating, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	clusterUpdating.Status.Phase = constants.UPDATING
	clusterRW.EXPECT().GetCluster(gomock.Any(), "updating").Return(clusterUpdating, nil).AnyTimes()

	clusterDeleting, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	clusterDeleting.Status.Phase = constants.DELETING
	clusterRW.EXPECT().GetCluster(gomock.Any(), "deleting").Return(clusterDeleting, nil).AnyTimes()

	t.Run("id not found", func(t *testing.T) {
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID: "not found",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})
	t.Run("cluster is deleting - conflict", func(t *testing.T) {
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID: "deleting",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterStatusConflict, err.(errors.Error).Code())
	})
	t.Run("update failed", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID:        "validID",
			MuteAlert: pointer.Bool(true),
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})
	t.Run("update mute alert", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID:        "validID",
			MuteAlert: pointer.Bool(true),
		})
		assert.NoError(t, err)
	})
	t.Run("update rule template id", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID:             "validID",
			RuleTemplateID: "new-template-id",
		})
		assert.NoError(t, err)
	})
	t.Run("update monitor spec id", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID:            "validID",
			MonitorSpecID: "new-spec-id",
		})
		assert.NoError(t, err)
	})
	t.Run("update all fields", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateCluster(context.TODO(), message.UpdateClusterReq{
			ID:             "validID",
			MuteAlert:      pointer.Bool(false),
			RuleTemplateID: "new-template-id",
			MonitorSpecID:  "new-spec-id",
		})
		assert.NoError(t, err)
	})
}

func TestClusterHandler_QueryClusters(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	req1 := mockCreateReq()
	req1.ID = utils.GenerateID()
	cluster1, _ := BuildClusterResourceRequirements(context.TODO(), req1, "", nil)

	req2 := mockCreateReq()
	req2.ID = utils.GenerateID()
	cluster2, _ := BuildClusterResourceRequirements(context.TODO(), req2, "", nil)

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().
			QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
			Return(&v1.ClusterList{ListMeta: metav1.ListMeta{Continue: "ccc"}, Items: []v1.Cluster{*cluster1, *cluster2}}, nil).Times(1)
		resp, err := GetHandler().QueryClusters(context.TODO(), message.QueryClustersReq{})
		assert.NoError(t, err)
		assert.Equal(t, 2, len(resp.Clusters))
		assert.Equal(t, "ccc", resp.ContinueToken)
	})
	t.Run("query failed", func(t *testing.T) {
		clusterRW.EXPECT().
			QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
			Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().QueryClusters(context.TODO(), message.QueryClustersReq{})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})
}

func TestClusterHandler_QueryClustersWithStorage(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	serviceInfo := originalService
	serviceInfo.LogicalRegionName = defaultRegion
	configs.Init(serviceInfo)

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	createReq := mockCreateReq()
	createReq.ID = utils.GenerateID()
	cluster, _ := BuildClusterResourceRequirements(context.TODO(), createReq, "", nil)

	clusterRW.EXPECT().QueryClusters(
		gomock.Any(),
		createReq.TenantID,
		createReq.ProjectID,
		createReq.ApplicationID,
		createReq.BizType,
		gomock.Any(),
	).Return(&v1.ClusterList{
		ListMeta: metav1.ListMeta{Continue: "ccc"},
		Items:    []v1.Cluster{*cluster},
	}, nil).Times(1)

	objectStorageRW.EXPECT().
		GetObjectStorage(gomock.Any(), gomock.Any()).
		Times(0)

	resp, err := GetHandler().QueryClustersWithStorage(context.TODO(), message.QueryClustersReq{
		CommonReqInfo: structs.CommonReqInfo{
			Vendor: createReq.Vendor,
			Region: createReq.Region,
		},
		TenantID:      createReq.TenantID,
		ProjectID:     createReq.ProjectID,
		ApplicationID: createReq.ApplicationID,
		BizType:       createReq.BizType,
	})

	assert.NoError(t, err)
	assert.Equal(t, "ccc", resp.ContinueToken)
	if assert.Len(t, resp.Clusters, 1) {
		storageByCategory := make(map[constants.ObservabilityCategory]structs.StorageInfo)
		for _, info := range resp.Clusters[0].StorageInfo {
			storageByCategory[info.Category] = info
		}
		assert.Len(t, resp.Clusters[0].StorageInfo, 4)
		logsInfo, ok := storageByCategory[constants.LOGS]
		if assert.True(t, ok) {
			assert.Equal(t, "0/t/p/a/logs/", logsInfo.ObjectPath)
			assert.Contains(t, logsInfo.DataARN, "/logs/")
		}
		assert.Equal(t, ObjectStorageResourceName(createReq.ClusterBaseInfo), storageByCategory[constants.SLOWQUERYS].BucketName)
		assert.Equal(t, "0/t/p/a/profiling/", storageByCategory[constants.PROFILING].ObjectPath)
		assert.Equal(t, "aaaa", storageByCategory[constants.SLOWQUERYS].StorageAccount)
		assert.Equal(t, "0/t/p/a/logs/slowlog/", storageByCategory[constants.SLOWQUERYS].ObjectPath)
		assert.Contains(t, storageByCategory[constants.SLOWQUERYS].DataARN, "/logs/slowlog/")
		assert.Equal(t, "aaaa", storageByCategory[constants.SQLSTATEMENTS].StorageAccount)
		assert.Equal(t, "0/t/p/a/logs/statementlog/", storageByCategory[constants.SQLSTATEMENTS].ObjectPath)
		assert.Contains(t, storageByCategory[constants.SQLSTATEMENTS].DataARN, "/logs/statementlog/")
		assert.Equal(t, "https://www.ds.us-east-2.aws.observability-dev.pingcap.cloud/loki/self-monitoring", resp.Clusters[0].LogQueryUri)
		assert.Empty(t, resp.Clusters[0].MultitenantQueryUri)
		assert.Equal(t, "https://www.ds.us-east-2.aws.observability-dev.pingcap.cloud/internal/diagnosis/api/v1/", resp.Clusters[0].DiagnosisQueryUri)
	}
}

func TestClusterHandler_QueryClustersWithStorage_DoesNotLookupObjectStorage(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	createReq := mockCreateReq()
	createReq.ID = utils.GenerateID()
	cluster, _ := BuildClusterResourceRequirements(context.TODO(), createReq, "", nil)

	clusterRW.EXPECT().QueryClusters(
		gomock.Any(),
		createReq.TenantID,
		createReq.ProjectID,
		createReq.ApplicationID,
		createReq.BizType,
		gomock.Any(),
	).Return(&v1.ClusterList{
		Items: []v1.Cluster{*cluster},
	}, nil).Times(1)

	objectStorageRW.EXPECT().
		GetObjectStorage(gomock.Any(), gomock.Any()).
		Times(0)

	resp, err := GetHandler().QueryClustersWithStorage(context.TODO(), message.QueryClustersReq{
		CommonReqInfo: structs.CommonReqInfo{
			Vendor: createReq.Vendor,
			Region: createReq.Region,
		},
		TenantID:      createReq.TenantID,
		ProjectID:     createReq.ProjectID,
		ApplicationID: createReq.ApplicationID,
		BizType:       createReq.BizType,
	})

	assert.NoError(t, err)
	assert.Len(t, resp.Clusters, 1)
}

func TestClusterHandler_QueryClustersWithStorage_ShowDeleted(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	configs.Init(structs.ServiceInfo{
		ServerName:        constants.ObservabilityRegionalServer,
		LogicalRegionName: "us-east-1",
		Vendor:            "aws",
		Region:            "us-east-1",
		Env:               constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName: "www.ds.aws.observability-dev.pingcap.cloud",
			},
		},
	})

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	t.Run("showDeleted requires exact identity", func(t *testing.T) {
		_, err := GetHandler().QueryClustersWithStorage(context.TODO(), message.QueryClustersReq{
			ShowDeleted: true,
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())
	})

	t.Run("showDeleted falls back to archived metadata", func(t *testing.T) {
		archivedRW := mockresourcemanager.NewMockArchivedClusterReaderWriter(ctrl)
		resourcemanager.MockArchivedClusterReaderWriter(archivedRW)

		req := message.QueryClustersReq{
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AWS,
				Region: "us-east-1",
			},
			TenantID:      "tenant-archived-1",
			ProjectID:     "project-archived-1",
			ApplicationID: "app-archived-1",
			BizType:       "tidbcloud",
			ShowDeleted:   true,
		}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-archived-1",
			"project-archived-1",
			"app-archived-1",
			"tidbcloud",
			gomock.Any(),
		).Return(&v1.ClusterList{}, nil).Times(1)

		archivedRW.EXPECT().Get(
			gomock.Any(),
			gomock.Any(),
			structs.ClusterUniqueKey{
				BizType:       "tidbcloud",
				TenantID:      "tenant-archived-1",
				ProjectID:     "project-archived-1",
				ApplicationID: "app-archived-1",
			},
		).Return(structs.ArchivedClusterInfo{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				CommonReqInfo: structs.CommonReqInfo{
					Vendor: constants.AWS,
					Region: "us-east-1",
				},
				TenantID:      "tenant-archived-1",
				ProjectID:     "project-archived-1",
				ApplicationID: "app-archived-1",
				BizType:       "tidbcloud",
				ShareMode:     constants.SHARED,
			},
			VMAccountID: "1212",
			StorageInfo: []structs.StorageInfo{
				{
					Category: constants.SLOWQUERYS,
				},
				{
					Category: constants.SQLSTATEMENTS,
				},
			},
		}, nil).Times(1)

		recorder := httptest.NewRecorder()
		ginCtx, _ := gin.CreateTestContext(recorder)
		ginCtx.Request = httptest.NewRequest(http.MethodGet, "https://regional.example.com/api/v1/clusters?showDeleted=true", nil)

		resp, err := GetHandler().QueryClustersWithStorage(ginCtx, req)
		assert.NoError(t, err)
		if assert.Len(t, resp.Clusters, 1) {
			assert.Equal(t, "app-archived-1", resp.Clusters[0].ApplicationID)
			assert.Len(t, resp.Clusters[0].StorageInfo, 4)
			assert.Equal(
				t,
				"https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/historical/metrics/api/v1/tidbcloud/orgs/tenant-archived-1/projects/project-archived-1/clusters/app-archived-1/prometheus/",
				resp.Clusters[0].InternalReadUri,
			)
			storageByCategory := make(map[constants.ObservabilityCategory]structs.StorageInfo)
			for _, info := range resp.Clusters[0].StorageInfo {
				storageByCategory[info.Category] = info
			}
			logsInfo, ok := storageByCategory[constants.LOGS]
			if assert.True(t, ok) {
				assert.Equal(t, "0/tenant-archived-1/project-archived-1/app-archived-1/logs/", logsInfo.ObjectPath)
				assert.Contains(t, logsInfo.DataARN, "/logs/")
			}
			assert.Equal(t, "0/tenant-archived-1/project-archived-1/app-archived-1/profiling/", storageByCategory[constants.PROFILING].ObjectPath)
			assert.Equal(t, "0/tenant-archived-1/project-archived-1/app-archived-1/logs/slowlog/", storageByCategory[constants.SLOWQUERYS].ObjectPath)
			assert.Equal(t, "0/tenant-archived-1/project-archived-1/app-archived-1/logs/statementlog/", storageByCategory[constants.SQLSTATEMENTS].ObjectPath)
			assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/loki/self-monitoring", resp.Clusters[0].LogQueryUri)
			assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/v2/metrics/multitenant/api/v1/?extra_filters%5B%5D=%7Bvm_account_id%3D%221212%22%7D", resp.Clusters[0].MultitenantQueryUri)
			assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/diagnosis/api/v1/", resp.Clusters[0].DiagnosisQueryUri)
		}
	})

	t.Run("showDeleted returns empty list when archive is missing", func(t *testing.T) {
		archivedRW := mockresourcemanager.NewMockArchivedClusterReaderWriter(ctrl)
		resourcemanager.MockArchivedClusterReaderWriter(archivedRW)

		req := message.QueryClustersReq{
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AWS,
				Region: "us-east-1",
			},
			TenantID:      "tenant-archived-2",
			ProjectID:     "project-archived-2",
			ApplicationID: "app-archived-2",
			BizType:       "tidbcloud",
			ShowDeleted:   true,
		}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-archived-2",
			"project-archived-2",
			"app-archived-2",
			"tidbcloud",
			gomock.Any(),
		).Return(&v1.ClusterList{}, nil).Times(1)

		archivedRW.EXPECT().Get(
			gomock.Any(),
			gomock.Any(),
			structs.ClusterUniqueKey{
				BizType:       "tidbcloud",
				TenantID:      "tenant-archived-2",
				ProjectID:     "project-archived-2",
				ApplicationID: "app-archived-2",
			},
		).Return(structs.ArchivedClusterInfo{}, errors.New(errors.ObjectNotFoundError, "")).Times(1)

		resp, err := GetHandler().QueryClustersWithStorage(context.TODO(), req)
		assert.NoError(t, err)
		assert.Empty(t, resp.Clusters)
	})

	t.Run("showDeleted historical uri omits project path when project id is empty", func(t *testing.T) {
		archivedRW := mockresourcemanager.NewMockArchivedClusterReaderWriter(ctrl)
		resourcemanager.MockArchivedClusterReaderWriter(archivedRW)

		req := message.QueryClustersReq{
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AWS,
				Region: "us-east-1",
			},
			TenantID:      "tenant-archived-3",
			ApplicationID: "app-archived-3",
			BizType:       "tidbcloud",
			ShowDeleted:   true,
		}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-archived-3",
			"",
			"app-archived-3",
			"tidbcloud",
			gomock.Any(),
		).Return(&v1.ClusterList{}, nil).Times(1)

		archivedRW.EXPECT().Get(
			gomock.Any(),
			gomock.Any(),
			structs.ClusterUniqueKey{
				BizType:       "tidbcloud",
				TenantID:      "tenant-archived-3",
				ApplicationID: "app-archived-3",
			},
		).Return(structs.ArchivedClusterInfo{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				CommonReqInfo: structs.CommonReqInfo{
					Vendor: constants.AWS,
					Region: "us-east-1",
				},
				TenantID:      "tenant-archived-3",
				ApplicationID: "app-archived-3",
				BizType:       "tidbcloud",
				ShareMode:     constants.SHARED,
			},
			VMAccountID: "3434",
			StorageInfo: []structs.StorageInfo{
				{
					Category: constants.SLOWQUERYS,
				},
			},
		}, nil).Times(1)

		resp, err := GetHandler().QueryClustersWithStorage(context.TODO(), req)
		assert.NoError(t, err)
		if assert.Len(t, resp.Clusters, 1) {
			assert.Equal(
				t,
				"https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/historical/metrics/api/v1/tidbcloud/orgs/tenant-archived-3/clusters/app-archived-3/prometheus/",
				resp.Clusters[0].InternalReadUri,
			)
			assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/v2/metrics/multitenant/api/v1/?extra_filters%5B%5D=%7Bvm_account_id%3D%223434%22%7D", resp.Clusters[0].MultitenantQueryUri)
		}
	})

	t.Run("showDeleted keeps active rows and skips archive", func(t *testing.T) {
		archivedRW := mockresourcemanager.NewMockArchivedClusterReaderWriter(ctrl)
		resourcemanager.MockArchivedClusterReaderWriter(archivedRW)

		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "active-cluster-1",
				Labels: map[string]string{
					constants.LabelVendor:        "aws",
					constants.LabelRegion:        "us-east-1",
					constants.LabelTenantID:      "tenant-active-1",
					constants.LabelProjectID:     "project-active-1",
					constants.LabelApplicationID: "app-active-1",
					constants.LabelBizType:       "tidbcloud",
					constants.LabelShareMode:     string(constants.SHARED),
				},
			},
			Spec: v1.ClusterSpec{
				MuteAlert: false,
			},
		}
		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-active-1",
			"project-active-1",
			"app-active-1",
			"tidbcloud",
			gomock.Any(),
		).Return(&v1.ClusterList{
			Items: []v1.Cluster{*cluster},
		}, nil).Times(1)

		objectStorageRW.EXPECT().
			GetObjectStorage(gomock.Any(), gomock.Any()).
			Times(0)

		archivedRW.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Times(0)

		resp, err := GetHandler().QueryClustersWithStorage(context.TODO(), message.QueryClustersReq{
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AWS,
				Region: "us-east-1",
			},
			TenantID:      "tenant-active-1",
			ProjectID:     "project-active-1",
			ApplicationID: "app-active-1",
			BizType:       "tidbcloud",
			ShowDeleted:   true,
		})
		assert.NoError(t, err)
		assert.Len(t, resp.Clusters, 1)
		assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/loki/self-monitoring", resp.Clusters[0].LogQueryUri)
	})
}

func TestClusterHandler_QueryClustersWithStorage_SharedByocURIFields(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	serviceInfo := originalService
	serviceInfo.LogicalRegionName = "us-east-1"
	serviceInfo.Vendor = string(constants.AWS)
	serviceInfo.Region = "us-east-1"
	serviceInfo.Env = constants.EnvDev
	serviceInfo.BizTypeConfigs = map[string]structs.BizTypeConfig{
		constants.DefaultByocBizType: {
			O11yGlobalDSHostName: "www.ds.aws.observability-dev.pingcap.cloud",
		},
	}
	configs.Init(serviceInfo)

	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name: "byoc-cluster-1",
			Labels: map[string]string{
				constants.LabelVendor:        "aws",
				constants.LabelRegion:        "us-east-1",
				constants.LabelTenantID:      "tenant-byoc-1",
				constants.LabelProjectID:     "project-byoc-1",
				constants.LabelApplicationID: "app-byoc-1",
				constants.LabelBizType:       constants.DefaultByocBizType,
			},
			Annotations: map[string]string{
				constants.AnnotationShareMode: string(constants.SHARED),
			},
		},
		Spec: v1.ClusterSpec{
			VictoriaAccount: v1.VictoriaAccount{
				AccountID: 1212,
			},
		},
	}

	clusterRW.EXPECT().QueryClusters(
		gomock.Any(),
		"tenant-byoc-1",
		"project-byoc-1",
		"app-byoc-1",
		constants.DefaultByocBizType,
		gomock.Any(),
	).Return(&v1.ClusterList{
		Items: []v1.Cluster{*cluster},
	}, nil).Times(1)

	resp, err := GetHandler().QueryClustersWithStorage(context.TODO(), message.QueryClustersReq{
		CommonReqInfo: structs.CommonReqInfo{
			Vendor: constants.AWS,
			Region: "us-east-1",
		},
		TenantID:      "tenant-byoc-1",
		ProjectID:     "project-byoc-1",
		ApplicationID: "app-byoc-1",
		BizType:       constants.DefaultByocBizType,
	})
	assert.NoError(t, err)
	if assert.Len(t, resp.Clusters, 1) {
		assert.Equal(t, "https://www.ds.aws.observability-dev.pingcap.cloud/internal/byoc-cluster-1/metrics/api/v1/", resp.Clusters[0].InternalReadUri)
		assert.Equal(t, "https://www.ds.aws.observability-dev.pingcap.cloud/internal/loki/self-monitoring", resp.Clusters[0].LogQueryUri)
		assert.Equal(t, "https://www.ds.aws.observability-dev.pingcap.cloud/internal/diagnosis/api/v1/", resp.Clusters[0].DiagnosisQueryUri)
		assert.Equal(
			t,
			"https://www.ds.aws.observability-dev.pingcap.cloud/internal/multitenant/metrics/api/v1/?extra_filters%5B%5D=%7Bvm_account_id%3D%221212%22%7D",
			resp.Clusters[0].MultitenantQueryUri,
		)
	}
}

func TestClusterHandler_MonitorClusterSpec(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	req1 := mockCreateReq()
	req1.ID = utils.GenerateID()
	cluster1, _ := BuildClusterResourceRequirements(context.TODO(), req1, "", nil)

	reqShared := mockSharedReq()
	reqShared.ID = utils.GenerateID()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "parentID").Return(&v1.Cluster{
		Spec: v1.ClusterSpec{
			VictoriaAccount: v1.VictoriaAccount{
				AccountID: 999,
				ProjectID: 0,
			},
		},
	}, nil).Times(1)
	clusterShared, _ := BuildClusterResourceRequirements(context.TODO(), reqShared, "", nil)

	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	configs.Init(structs.ServiceInfo{
		MonitorSpec: map[string]structs.MonitorSpec{
			"default": {
				Cpu:                 "10m",
				Memory:              "200Mi",
				InitialStorage:      "40Gi",
				OverSell:            1,
				IsDefault:           true,
				MaxQueryConcurrency: pointer.Int32(10),
				MaxQuerySamples:     pointer.Int32(10000),
			},
			"vmalert.default": {
				Cpu:            "10m",
				Memory:         "200Mi",
				InitialStorage: "40Gi",
				OverSell:       1,
				MaxQueueSize:   pointer.Int32(10000),
			},
		},
	})

	clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(cluster1, nil).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), reqShared.ID).Return(clusterShared, nil).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "not found").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()

	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     req1.ID,
			SpecID: "default",
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, cluster1.Spec.Prometheus.Query)
		assert.Equal(t, *cluster1.Spec.Prometheus.Query.MaxConcurrency, int32(10))
	})

	t.Run("vmalert-normal", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     reqShared.ID,
			SpecID: "vmalert.default",
		})
		assert.NoError(t, err)
		assert.Equal(t, *clusterShared.Spec.VictoriaAccount.MaxQueueSize, int32(10000))
	})

	t.Run("invalid cluster id", func(t *testing.T) {
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     "not found",
			SpecID: "default",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})
	t.Run("invalid spec", func(t *testing.T) {
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     req1.ID,
			SpecID: "unsupported",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnsupportedMonitorSpec, err.(errors.Error).Code())
	})
	t.Run("update failed", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     req1.ID,
			SpecID: "default",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})
}

func TestClusterHandler_MonitorClusterSpec_Shared(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	dataAccountRW := mockresourcemanager.NewMockDataAccountReaderWriter(ctrl)
	resourcemanager.MockDataAccountReaderWriter(dataAccountRW)

	dataAccountRW.EXPECT().UniqueInt32ForCluster(gomock.Any()).Return(int32(1212), nil).AnyTimes()

	req1 := mockCreateReq()
	req1.ID = "not allowed"
	req1.ShareMode = constants.SHARED
	cluster1, err := BuildClusterResourceRequirements(context.TODO(), req1, "", nil)
	assert.NoError(t, err)
	assert.NotNil(t, cluster1)

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

}

func Test_prepareFailed(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	t.Run("IsObjectStorageExisted failed", func(t *testing.T) {
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(false, errors.New(errors.UnrecognizedError, "")).Times(1)
		err := prepareBucketResource(context.TODO(), message.CreateClusterReq{})
		assert.Error(t, err)
	})

}

func Test_parseFailed(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	t.Run("parseClusterBaseInfo invalid parameter", func(t *testing.T) {
		_, err := parseClusterBaseInfo(context.TODO(), nil)
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())
	})

	t.Run("parseMonitorInfo invalid parameter", func(t *testing.T) {
		_, err := parseMonitorInfo(context.TODO(), nil)
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())
	})
	t.Run("parseMonitorInfo az empty", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			AZConfig: map[string]structs.AvailabilityZoneConfig{},
		})
		_, err := parseMonitorInfo(context.TODO(), &v1.Cluster{})
		assert.Error(t, err)
		assert.Equal(t, errors.AbsenceOfConfig, err.(errors.Error).Code())
	})
	t.Run("parseMonitorInfo no az matched", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			AZConfig: map[string]structs.AvailabilityZoneConfig{
				"a": {},
			},
		})
		_, err := parseMonitorInfo(context.TODO(), &v1.Cluster{})
		assert.Error(t, err)
		assert.Equal(t, errors.AbsenceOfConfig, err.(errors.Error).Code())
	})

	t.Run("parseStorageInfo invalid parameter", func(t *testing.T) {
		_, err := parseStorageInfo(context.TODO(), nil, nil)
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())
	})
	t.Run("parseStorageInfo does not lookup object storage", func(t *testing.T) {
		objectStorageRW.EXPECT().GetObjectStorage(gomock.Any(), gomock.Any()).Times(0)
		storageInfo, err := parseStorageInfo(context.TODO(), &structs.ClusterBaseInfo{
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AWS,
				Region: defaultRegion,
			},
			TenantID:      "tenant-storage-1",
			ProjectID:     "project-storage-1",
			ApplicationID: "app-storage-1",
			BizType:       "tidbcloud",
		}, &v1.Cluster{})
		assert.NoError(t, err)
		assert.NotEmpty(t, storageInfo)
	})

	t.Run("parseAgentInfo invalid parameter", func(t *testing.T) {
		_, err := parseAgentInfo(context.TODO(), nil, nil)
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())

	})

	t.Run("parseClusterStatus invalid parameter", func(t *testing.T) {
		_, err := parseClusterStatus(context.TODO(), nil)
		assert.Error(t, err)
		assert.Equal(t, errors.ParameterInvalid, err.(errors.Error).Code())
	})

}

func TestClusterHandler_GetClusterDetail(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	clusterRW.EXPECT().GetCluster(gomock.Any(), "notFound").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()

	validCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
	validCluster.Name = "validID"
	clusterRW.EXPECT().GetCluster(gomock.Any(), "validID").Return(validCluster, nil).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "parentID").Return(&v1.Cluster{
		Spec: v1.ClusterSpec{
			VictoriaAccount: v1.VictoriaAccount{
				AccountID: 999,
				ProjectID: 0,
			},
		},
	}, nil).AnyTimes()

	sharedCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockSharedReq(), "", nil)
	sharedCluster.Name = "validShared"
	clusterRW.EXPECT().GetCluster(gomock.Any(), "validShared").Return(sharedCluster, nil).AnyTimes()

	clusterRW.EXPECT().GetCluster(gomock.Any(), "empty").Return(nil, nil).AnyTimes()

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	t.Run("404", func(t *testing.T) {
		_, err := GetHandler().GetClusterDetail(context.TODO(), message.DetailClusterReq{
			ID: "notFound",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})
	t.Run("empty", func(t *testing.T) {
		_, err := GetHandler().GetClusterDetail(context.TODO(), message.DetailClusterReq{
			ID: "empty",
		})
		assert.Error(t, err)
	})
	t.Run("normal", func(t *testing.T) {

		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		resp, err := GetHandler().GetClusterDetail(context.TODO(), message.DetailClusterReq{
			ID: "validID",
		})
		assert.NoError(t, err)
		assert.Equal(t, "validID", resp.ID)
	})
	t.Run("normal shared", func(t *testing.T) {

		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		resp, err := GetHandler().GetClusterDetail(context.TODO(), message.DetailClusterReq{
			ID: "validShared",
		})
		assert.NoError(t, err)
		assert.Equal(t, "validShared", resp.ID)

		assert.NotEmpty(t, resp.MonitorInfo.VMAccountID)
		assert.Equal(t, "999", resp.MonitorInfo.VMAccountID)

		assert.NotEmpty(t, resp.MonitorInfo.VMProjectID)
		assert.Equal(t, "777", resp.MonitorInfo.VMProjectID)
	})
}

func TestClusterHandler_UpdateAlertRules(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	req1 := mockCreateReq()
	req1.ID = utils.GenerateID()

	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	t.Run("normal", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example2"),
		})
		assert.NoError(t, err)
	})
	t.Run("same name for replace", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example"),
		})
		assert.NoError(t, err)
	})

	t.Run("overide", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
			Override:         true,
		})
		assert.NoError(t, err)
	})

	t.Run("get failed", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.Error(t, err)
	})
	t.Run("update failed", func(t *testing.T) {
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID: req1.ID,
		})
		assert.Error(t, err)
	})
	t.Run("cluster not found", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(nil, errors.New(errors.ClusterNotFound, "cluster not found")).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.Error(t, err)
	})

	t.Run("binding cluster", func(t *testing.T) {
		returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(returnCluster, nil).Times(1)
		promeRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.NoError(t, err)
	})

	t.Run("create new prometheusrule", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"default": {
					BizType:          "b",
					MultiTenancyRule: true,
				},
			},
		})
		returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(returnCluster, nil).Times(1)
		promeRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		promeRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			BizType:          "b",
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.NoError(t, err)
	})

	t.Run("create new prometheusrule fail", func(t *testing.T) {
		returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(returnCluster, nil).Times(1)
		promeRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		promeRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.Error(t, err)
	})

	t.Run("create new prometheusrule fail", func(t *testing.T) {
		returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(returnCluster, nil).Times(1)
		promeRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		promeRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.Error(t, err)
	})

	t.Run("binding cluster fail", func(t *testing.T) {
		returnCluster, _ := BuildClusterResourceRequirements(context.TODO(), mockCreateReq(), "", nil)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(returnCluster, nil).Times(1)
		promeRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().UpdateAlertRules(context.TODO(), message.UpdateAlertRulesReq{
			ID:               req1.ID,
			MonitorPromRules: mockPrometheusRuleSpec("example1"),
		})
		assert.Error(t, err)
	})
}

func TestClusterHandler_QueryAlertRules(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	req1 := mockCreateReq()
	req1.ID = utils.GenerateID()

	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	t.Run("normal", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		resp, err := promeRuleRW.GetPrometheusRule(context.TODO(), "id")
		assert.NoError(t, err)
		assert.Equal(t, 1, len(resp.Spec.Groups))
	})
	t.Run("get failed", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := promeRuleRW.GetPrometheusRule(context.TODO(), "id")
		assert.Error(t, err)
	})
	t.Run("not exists", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		_, err := promeRuleRW.GetPrometheusRule(context.TODO(), "id")
		assert.Error(t, err)
	})
}

func TestClusterHandler_ModifyAlertRulesTemplate(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	t.Run("normal", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().ModifyAlertRulesTemplate(context.TODO(), message.ModifyAlertRulesTemplateReq{
			TemplateID:       "test",
			MonitorPromRules: mockPrometheusRuleSpec("example2"),
		})
		assert.NoError(t, err)
	})
	t.Run("rule empty", func(t *testing.T) {
		_, err := GetHandler().ModifyAlertRulesTemplate(context.TODO(), message.ModifyAlertRulesTemplateReq{
			TemplateID: "test",
		})
		assert.NotNil(t, err)
	})

	t.Run("query error", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().ModifyAlertRulesTemplate(context.TODO(), message.ModifyAlertRulesTemplateReq{
			TemplateID:       "test",
			MonitorPromRules: mockPrometheusRuleSpec("example2"),
		})
		assert.NotNil(t, err)
	})

	t.Run("create succ", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PrometheusRuleNotFound, "")).Times(1)
		promeRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().ModifyAlertRulesTemplate(context.TODO(), message.ModifyAlertRulesTemplateReq{
			TemplateID:       "test",
			MonitorPromRules: mockPrometheusRuleSpec("example2"),
		})
		assert.NoError(t, err)
	})
}

func TestClusterHandler_QueryAlertRulesTemplate(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	t.Run("normal", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		_, err := GetHandler().QueryAlertRuleTemplate(context.TODO(), message.QueryAlertRulesTemplateReq{
			ID: "test",
		})
		assert.NoError(t, err)
	})
	t.Run("get failed", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().QueryAlertRuleTemplate(context.TODO(), message.QueryAlertRulesTemplateReq{
			ID: "test",
		})
		assert.NotNil(t, err)
	})
}

func TestClusterHandler_DeleteAlertRulesTemplate(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	t.Run("normal", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		promeRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().DeleteAlertRulesTemplate(context.TODO(), message.DeleteAlertRulesTemplateReq{
			ID: "test",
		})
		assert.NoError(t, err)
	})
	t.Run("delete failed", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).Times(1)
		promeRuleRW.EXPECT().DeletePrometheusRules(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().DeleteAlertRulesTemplate(context.TODO(), message.DeleteAlertRulesTemplateReq{
			ID: "test",
		})
		assert.NotNil(t, err)
	})

	t.Run("query failed", func(t *testing.T) {
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().DeleteAlertRulesTemplate(context.TODO(), message.DeleteAlertRulesTemplateReq{
			ID: "test",
		})
		assert.NotNil(t, err)
	})
}

func Test_ModifyMonitorStorage(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	namespaceRW := mockresourcemanager.NewMockNamespaceReaderWriter(ctrl)
	resourcemanager.MockNamespaceReaderWriter(namespaceRW)
	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).AnyTimes()
	promRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes()
	promRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
	pvcRW := mockresourcemanager.NewMockPVCReaderWriter(ctrl)
	resourcemanager.MockPVCReaderWriter(pvcRW)
	req := mockCreateReq()
	namespaceMock := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: NamespaceName(req.BizType, req.TenantID),
			Labels: map[string]string{
				constants.LabelBizType:  req.BizType,
				constants.LabelTenantID: req.TenantID,
			},
		},
	}
	t.Run("shared cluster", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), gomock.Any()).Return(&v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					constants.AnnotationShareMode: string(constants.SHARED),
				},
			},
		}, nil).Times(1)
		_, err := GetHandler().ModifyMonitorStorage(context.TODO(), message.ModifyMonitorStorageReq{
			ID: "ttt",
		})
		assert.Error(t, err)
	})
	t.Run("failed to get cluster", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.ClusterNotFound, "")).Times(1)
		_, err := GetHandler().ModifyMonitorStorage(context.TODO(), message.ModifyMonitorStorageReq{
			ID: "ttt",
		})
		assert.Error(t, err)
	})
	t.Run("failed to get pvc", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), gomock.Any()).Return(&v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					constants.AnnotationShareMode: string(constants.EXCLUSIVE),
				},
			},
		}, nil).Times(1)
		pvcRW.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, errors.New(errors.PVCNotFound, ""))
		_, err := GetHandler().ModifyMonitorStorage(context.TODO(), message.ModifyMonitorStorageReq{
			ID: "ttt",
		})
		assert.Error(t, err)
	})
	t.Run("failed to update pvc", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), gomock.Any()).Return(&v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					constants.AnnotationShareMode: string(constants.EXCLUSIVE),
				},
			},
		}, nil).Times(1)
		pvcRW.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&corev1.PersistentVolumeClaim{
			Spec: corev1.PersistentVolumeClaimSpec{
				Resources: corev1.VolumeResourceRequirements{
					Requests: map[corev1.ResourceName]resource.Quantity{},
				},
			},
		}, nil).Times(1)
		pvcRW.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		//clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)

		_, err := GetHandler().ModifyMonitorStorage(context.TODO(), message.ModifyMonitorStorageReq{
			ID: "ttt",
		})
		assert.Error(t, err)
	})

	t.Run("failed to update cluster", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), gomock.Any()).Return(&v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					constants.AnnotationShareMode: string(constants.EXCLUSIVE),
				},
			},
		}, nil).Times(1)
		pvcRW.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&corev1.PersistentVolumeClaim{
			Spec: corev1.PersistentVolumeClaimSpec{
				Resources: corev1.VolumeResourceRequirements{
					Requests: map[corev1.ResourceName]resource.Quantity{},
				},
			},
		}, nil).Times(1)
		pvcRW.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)

		_, err := GetHandler().ModifyMonitorStorage(context.TODO(), message.ModifyMonitorStorageReq{
			ID: "ttt",
		})
		assert.Error(t, err)
	})

	t.Run("normal", func(t *testing.T) {

		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
		namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
		objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
		iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
		iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		namespaceRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().CreateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelAlias: "alias",
				},
			},
		}, nil).Times(1)
		cluster, err := GetHandler().CreateCluster(context.TODO(), req)
		assert.NoError(t, err)
		cluster1, _ := BuildClusterResourceRequirements(context.TODO(), req, "", nil)

		clusterRW.EXPECT().GetCluster(gomock.Any(), cluster.ID).Return(cluster1, nil).Times(1)

		q, _ := resource.ParseQuantity("2Gi")
		pvcRW.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&corev1.PersistentVolumeClaim{
			Spec: corev1.PersistentVolumeClaimSpec{
				Resources: corev1.VolumeResourceRequirements{
					Requests: map[corev1.ResourceName]resource.Quantity{
						corev1.ResourceStorage: q,
					},
				},
			},
		}, nil).Times(1)
		pvcRW.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		_, err = GetHandler().ModifyMonitorStorage(context.TODO(), message.ModifyMonitorStorageReq{
			ID:        cluster.ID,
			Namespace: cluster1.Namespace,
			PVCName:   "",
			Request:   q,
		})
		assert.NoError(t, err)
	})
}

func Test_ModifyMonitorResource(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	objectStorageRW := mockresourcemanager.NewMockObjectStorageReaderWriter(ctrl)
	resourcemanager.MockObjectStorageReaderWriter(objectStorageRW)

	iamRW := mockresourcemanager.NewMockIAMReaderWriter(ctrl)
	resourcemanager.MockIAMReaderWriter(iamRW)

	namespaceRW := mockresourcemanager.NewMockNamespaceReaderWriter(ctrl)
	resourcemanager.MockNamespaceReaderWriter(namespaceRW)
	promRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promRuleRW)
	promRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(mockPrometheusRule("id", "test", "test"), nil).AnyTimes()
	promRuleRW.EXPECT().IsPrometheusRuleIDExists(gomock.Any(), gomock.Any()).Return(false, nil).AnyTimes()
	promRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	clusterRoleBindingRW := mockresourcemanager.NewMockClusterRoleBindingReaderWriter(ctrl)
	resourcemanager.MockClusterRoleBindingReaderWriter(clusterRoleBindingRW)
	clusterRoleBindingRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
	pvcRW := mockresourcemanager.NewMockPVCReaderWriter(ctrl)
	resourcemanager.MockPVCReaderWriter(pvcRW)
	req := mockCreateReq()
	namespaceMock := &corev1.Namespace{
		ObjectMeta: metav1.ObjectMeta{
			Name: NamespaceName(req.BizType, req.TenantID),
			Labels: map[string]string{
				constants.LabelBizType:  req.BizType,
				constants.LabelTenantID: req.TenantID,
			},
		},
	}

	clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.ClusterList{}, nil).Times(1)
	namespaceRW.EXPECT().Get(gomock.Any(), gomock.Any()).Return(namespaceMock, nil).Times(1)
	objectStorageRW.EXPECT().IsObjectStorageExisted(gomock.Any(), gomock.Any()).Return(true, nil).Times(1)
	iamRW.EXPECT().IsRoleExists(gomock.Any(), gomock.Any(), gomock.Any()).Return(false, nil).Times(1)
	iamRW.EXPECT().CreateRole(gomock.Any(), gomock.Any()).Return(nil).Times(1)
	namespaceRW.EXPECT().CreateServiceAccount(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
	clusterRW.EXPECT().CreateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
	returnS3, _ := BuildObjectStorageResourceRequirements(ObjectStorageResourceName(req.ClusterBaseInfo))
	objectStorageRW.EXPECT().GetObjectStorage(gomock.Any(), gomock.Any()).Return(returnS3, nil).AnyTimes()
	iamRW.EXPECT().GetRole(gomock.Any(), gomock.Any(), gomock.Any()).Return(&v1.IAM{
		ObjectMeta: metav1.ObjectMeta{
			Labels: map[string]string{
				constants.LabelAlias: "alias",
			},
		},
	}, nil).AnyTimes()
	cluster, err := GetHandler().CreateCluster(context.TODO(), req)
	assert.NoError(t, err)
	cluster1, _ := BuildClusterResourceRequirements(context.TODO(), req, "", nil)

	t.Run("failed to get cluster", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), cluster.ID).Return(nil, errors.NewErrorf(errors.ClusterNotFound, "")).Times(1)
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceMemory,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceMemory: {
					SpecID: "spec1",
				},
			},
		})
		assert.Error(t, err)
	})

	clusterRW.EXPECT().GetCluster(gomock.Any(), cluster.ID).Return(cluster1, nil).AnyTimes()
	clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()

	t.Run("update cpu", func(t *testing.T) {
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceCPU,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceCPU: {
					SpecID: "spec1",
				},
			},
		})
		assert.NoError(t, err)

		_, err = GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     cluster.ID,
			SpecID: "default",
		})
	})
	t.Run("update memory", func(t *testing.T) {
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceMemory,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceMemory: {
					SpecID: "spec1",
				},
			},
		})
		assert.NoError(t, err)
	})
	t.Run("update storage", func(t *testing.T) {
		q, _ := resource.ParseQuantity("2Gi")
		pvcRW.EXPECT().Get(gomock.Any(), gomock.Any(), gomock.Any()).Return(&corev1.PersistentVolumeClaim{
			Spec: corev1.PersistentVolumeClaimSpec{
				Resources: corev1.VolumeResourceRequirements{
					Requests: map[corev1.ResourceName]resource.Quantity{
						corev1.ResourceStorage: q,
					},
				},
			},
		}, nil).Times(2)
		pvcRW.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceStorage,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceStorage: {
					Namespace: "ns",
					PVCName:   "name",
					//Request:   q,
				},
			},
		})
		assert.NoError(t, err)
	})
	t.Run("update fail", func(t *testing.T) {
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceCPU,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceCPU: {},
			},
		})
		assert.Error(t, err)
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceMemory,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceMemory: {},
			},
		})
		assert.Error(t, err)
		_, err = GetHandler().ModifyMonitorResource(context.TODO(), message.ModifyMonitorResourceReq{
			ID:           cluster.ID,
			ResourceType: corev1.ResourceStorage,
			ResourceMeta: map[corev1.ResourceName]message.ResourceMeta{
				corev1.ResourceStorage: {},
			},
		})
		assert.Error(t, err)
	})

}

func TestClusterHandler_MuteClusterAlert(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	idNotFound := "notFound"
	clusterRW.EXPECT().GetCluster(gomock.Any(), idNotFound).Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()
	idFound := "idFound"
	clusterRW.EXPECT().GetCluster(gomock.Any(), idFound).Return(&v1.Cluster{
		Spec: v1.ClusterSpec{},
	}, nil).AnyTimes()

	t.Run("get cluster error", func(t *testing.T) {
		_, err := GetHandler().MuteClusterAlert(context.TODO(), message.MuteClusterAlertReq{
			ID: idNotFound,
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})

	t.Run("update error", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().MuteClusterAlert(context.TODO(), message.MuteClusterAlertReq{
			ID: idFound,
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})

	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().MuteClusterAlert(context.TODO(), message.MuteClusterAlertReq{
			ID: idFound,
		})
		assert.NoError(t, err)
	})

}

func TestClusterHandler_UnmuteClusterAlert(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	idNotFound := "notFound"
	clusterRW.EXPECT().GetCluster(gomock.Any(), idNotFound).Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()
	idFound := "idFound"
	clusterRW.EXPECT().GetCluster(gomock.Any(), idFound).Return(&v1.Cluster{
		Spec: v1.ClusterSpec{},
	}, nil).AnyTimes()

	t.Run("get cluster error", func(t *testing.T) {
		_, err := GetHandler().UnmuteClusterAlert(context.TODO(), message.UnmuteClusterAlertReq{
			ID: idNotFound,
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})

	t.Run("update error", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().UnmuteClusterAlert(context.TODO(), message.UnmuteClusterAlertReq{
			ID: idFound,
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})

	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().UnmuteClusterAlert(context.TODO(), message.UnmuteClusterAlertReq{
			ID: idFound,
		})
		assert.NoError(t, err)
	})

}

func Test_getCompatibleRequest(t *testing.T) {
	req := &message.CreateClusterReq{
		ClusterBaseInfo: structs.ClusterBaseInfo{
			ApplicationID: applicationIDDevTierInfraF,
		},
	}
	getCompatibleRequest(req)
	assert.Equal(t, true, req.MuteAlert)

	req = &message.CreateClusterReq{
		ClusterBaseInfo: structs.ClusterBaseInfo{
			BizType:       "forceToShare",
			ApplicationID: applicationIDDevTierInfraF,
		},
	}
	getCompatibleRequest(req)
	assert.Equal(t, true, req.MuteAlert)
}

func Test_dealuseRDS(t *testing.T) {
	original := configs.GetServiceInfo()
	defer configs.Init(original)

	t.Run("biztype with UseRDS enabled", func(t *testing.T) {
		// Setup: Configure a biztype with UseRDS=true
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test-rds-enabled": {
					BizType: "test-rds-enabled",
					UseRDS:  true,
				},
			},
		})

		ctx := context.Background()
		resultCtx := dealuseRDS(ctx, "test-rds-enabled")

		// Verify the context has UseRDSKey set to true
		value, ok := resultCtx.Value(constants.UseRDSKey).(bool)
		assert.True(t, ok, "UseRDSKey should be present in context")
		assert.True(t, value, "UseRDSKey should be true")
	})

	t.Run("biztype with UseRDS disabled", func(t *testing.T) {
		// Setup: Configure a biztype with UseRDS=false
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test-rds-disabled": {
					BizType: "test-rds-disabled",
					UseRDS:  false,
				},
			},
		})

		ctx := context.Background()
		resultCtx := dealuseRDS(ctx, "test-rds-disabled")

		// Verify the context does not have UseRDSKey set
		value, ok := resultCtx.Value(constants.UseRDSKey).(bool)
		assert.False(t, ok || value, "UseRDSKey should not be set or should be false")
	})

	t.Run("biztype not configured", func(t *testing.T) {
		// Setup: No biztype configs
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{},
		})

		ctx := context.Background()
		resultCtx := dealuseRDS(ctx, "non-existent-biztype")

		// Verify the context does not have UseRDSKey set
		value, ok := resultCtx.Value(constants.UseRDSKey).(bool)
		assert.False(t, ok || value, "UseRDSKey should not be set for non-existent biztype")
	})

	t.Run("context with existing values", func(t *testing.T) {
		// Setup: Configure a biztype with UseRDS=true
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test-context": {
					BizType: "test-context",
					UseRDS:  true,
				},
			},
		})

		// Create context with an existing value
		type testKey string
		const existingKey testKey = "existing"
		ctx := context.WithValue(context.Background(), existingKey, "test-value")

		resultCtx := dealuseRDS(ctx, "test-context")

		// Verify existing context values are preserved
		existingValue, ok := resultCtx.Value(existingKey).(string)
		assert.True(t, ok, "Existing context value should be preserved")
		assert.Equal(t, "test-value", existingValue)

		// Verify UseRDSKey is added
		useRDSValue, ok := resultCtx.Value(constants.UseRDSKey).(bool)
		assert.True(t, ok, "UseRDSKey should be present in context")
		assert.True(t, useRDSValue, "UseRDSKey should be true")
	})
}

func Test_checkuseRDS(t *testing.T) {
	t.Run("context with UseRDSKey set to true", func(t *testing.T) {
		ctx := context.WithValue(context.Background(), constants.UseRDSKey, true)
		result := checkuseRDS(ctx)
		assert.True(t, result, "checkuseRDS should return true when UseRDSKey is true")
	})

	t.Run("context with UseRDSKey set to false", func(t *testing.T) {
		ctx := context.WithValue(context.Background(), constants.UseRDSKey, false)
		result := checkuseRDS(ctx)
		assert.False(t, result, "checkuseRDS should return false when UseRDSKey is false")
	})

	t.Run("context without UseRDSKey", func(t *testing.T) {
		ctx := context.Background()
		result := checkuseRDS(ctx)
		assert.False(t, result, "checkuseRDS should return false when UseRDSKey is not set")
	})

	t.Run("context with UseRDSKey set to wrong type", func(t *testing.T) {
		ctx := context.WithValue(context.Background(), constants.UseRDSKey, "wrong-type")
		result := checkuseRDS(ctx)
		assert.False(t, result, "checkuseRDS should return false when UseRDSKey is not a bool")
	})

	t.Run("context with UseRDSKey set to nil", func(t *testing.T) {
		ctx := context.WithValue(context.Background(), constants.UseRDSKey, nil)
		result := checkuseRDS(ctx)
		assert.False(t, result, "checkuseRDS should return false when UseRDSKey is nil")
	})
}

func Test_dealuseRDS_and_checkuseRDS_integration(t *testing.T) {
	original := configs.GetServiceInfo()
	defer configs.Init(original)

	t.Run("complete flow with UseRDS enabled", func(t *testing.T) {
		// Setup
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"integration-test": {
					BizType: "integration-test",
					UseRDS:  true,
				},
			},
		})

		// Create context through dealuseRDS
		ctx := context.Background()
		ctx = dealuseRDS(ctx, "integration-test")

		// Check with checkuseRDS
		result := checkuseRDS(ctx)
		assert.True(t, result, "Integration test should return true for UseRDS enabled biztype")
	})

	t.Run("complete flow with UseRDS disabled", func(t *testing.T) {
		// Setup
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"integration-test-disabled": {
					BizType: "integration-test-disabled",
					UseRDS:  false,
				},
			},
		})

		// Create context through dealuseRDS
		ctx := context.Background()
		ctx = dealuseRDS(ctx, "integration-test-disabled")

		// Check with checkuseRDS
		result := checkuseRDS(ctx)
		assert.False(t, result, "Integration test should return false for UseRDS disabled biztype")
	})
}

func Test_prepareNamespace(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rw := mockresourcemanager.NewMockNamespaceReaderWriter(ctrl)
	resourcemanager.MockNamespaceReaderWriter(rw)

	dataAccountRW := mockresourcemanager.NewMockDataAccountReaderWriter(ctrl)
	resourcemanager.MockDataAccountReaderWriter(dataAccountRW)

	t.Run("error", func(t *testing.T) {
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(nil, errors.New(errors.UnrecognizedError, ""))
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{})
		assert.Error(t, err)
	})

	t.Run("ns exists", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "test",
				Labels: map[string]string{},
			},
		}, nil)
		rw.EXPECT().Update(context.TODO(), gomock.Any()).Return(nil).Times(1)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.NoError(t, err)
	})

	t.Run("ns exists with multi tenancy rule already set", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
				Labels: map[string]string{
					constants.LabelMultiTenancyRule: "true",
				},
			},
		}, nil)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.NoError(t, err)
	})

	t.Run("ns exists with agent next gen enabled and vm account exists", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
				Labels: map[string]string{
					constants.LabelMultiTenancyRule: "true",
					constants.LabelVMAccountID:      "12345",
				},
			},
		}, nil)
		vmAccount, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
			AgentNextGen: true,
		})
		assert.NoError(t, err)
		assert.Equal(t, "12345", vmAccount)
	})

	t.Run("ns exists with agent next gen enabled but vm account not exists", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
				Labels: map[string]string{
					constants.LabelMultiTenancyRule: "true",
				},
			},
		}, nil)
		dataAccountRW.EXPECT().UniqueInt32ForCluster(context.TODO()).Return(int32(54321), nil).Times(1)
		rw.EXPECT().Update(context.TODO(), gomock.Any()).Return(nil).Times(1)
		vmAccount, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
			AgentNextGen: true,
		})
		assert.NoError(t, err)
		assert.Equal(t, "54321", vmAccount)
	})

	t.Run("ns exists with agent next gen enabled but get unique int32 failed", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name: "test",
				Labels: map[string]string{
					constants.LabelMultiTenancyRule: "true",
				},
			},
		}, nil)
		dataAccountRW.EXPECT().UniqueInt32ForCluster(context.TODO()).Return(int32(0), errors.New(errors.UnrecognizedError, "failed to get unique int32")).Times(1)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
			AgentNextGen: true,
		})
		assert.Error(t, err)
	})

	t.Run("ns exists with exclusive share mode", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "test",
				Labels: map[string]string{},
			},
		}, nil)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.EXCLUSIVE,
				TenantID:  "123",
			},
		})
		assert.NoError(t, err)
	})

	t.Run("ns exists without multi tenancy rule config", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: false,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "test",
				Labels: map[string]string{},
			},
		}, nil)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.NoError(t, err)
	})

	t.Run("ns exists update failed", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer configs.Init(original)
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"test": {
					BizType:          "test",
					MultiTenancyRule: true,
				},
			},
		})
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(&corev1.Namespace{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "test",
				Labels: map[string]string{},
			},
		}, nil)
		rw.EXPECT().Update(context.TODO(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "update failed")).Times(1)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.Error(t, err)
	})

	t.Run("create", func(t *testing.T) {
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(nil, nil).Times(1)
		rw.EXPECT().Create(context.TODO(), gomock.Any()).Return(nil).Times(1)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.NoError(t, err)
	})

	t.Run("create with duplicated namespace error ignored", func(t *testing.T) {
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(nil, nil).Times(1)
		rw.EXPECT().Create(context.TODO(), gomock.Any()).Return(errors.New(errors.DuplicatedNamespace, "namespace already exists")).Times(1)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.NoError(t, err)
	})

	t.Run("create failed with other error", func(t *testing.T) {
		rw.EXPECT().Get(context.TODO(), gomock.Any()).Return(nil, nil).Times(1)
		rw.EXPECT().Create(context.TODO(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "create failed")).Times(1)
		_, err := prepareNamespace(context.TODO(), message.CreateClusterReq{
			ClusterBaseInfo: structs.ClusterBaseInfo{
				BizType:   "test",
				ShareMode: constants.SHARED,
				TenantID:  "123",
			},
		})
		assert.Error(t, err)
	})
}

func Test_preparePrometheusRuleResource(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	rw := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(rw)

	t.Run("error", func(t *testing.T) {
		rw.EXPECT().IsPrometheusRuleIDExists(context.TODO(), gomock.Any()).Return(false, errors.New(errors.UnrecognizedError, "")).Times(1)
		err := preparePrometheusRuleResource(context.TODO(), message.CreateClusterReq{}, &v1.Cluster{})
		assert.Error(t, err)
	})

	t.Run("error", func(t *testing.T) {
		rw.EXPECT().IsPrometheusRuleIDExists(context.TODO(), gomock.Any()).Return(true, nil).Times(1)
		rw.EXPECT().GetPrometheusRule(context.TODO(), gomock.Any()).Return(&prometheusv1.PrometheusRule{}, nil).Times(2)

		rw.EXPECT().UpdatePrometheusRule(context.TODO(), gomock.Any()).Return(nil).Times(1)
		err := preparePrometheusRuleResource(context.TODO(), message.CreateClusterReq{
			RuleTemplateID: "aaa",
		}, &v1.Cluster{
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					AccountID: 1234,
					ProjectID: 1,
				},
			}})
		assert.NoError(t, err)
	})

	t.Run("skip alert rule creation if use rds for serverless cluster", func(t *testing.T) {
		originalService := configs.GetServiceInfo()
		defer func() {
			configs.Init(originalService)
		}()
		configs.Init(structs.ServiceInfo{
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				"devtier": {
					BizType: "devtier",
					UseRDS:  true,
				},
			},
		})
		ctx := context.WithValue(context.TODO(), constants.UseRDSKey, true)
		err := preparePrometheusRuleResource(ctx, message.CreateClusterReq{
			ParentClusterID: "123",
			KeySpaceID:      "1",
		}, &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelBizType: "devtier",
				},
			},
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					AccountID: 1234,
					ProjectID: 1,
				},
				MuteAlert: true,
			}})
		assert.NoError(t, err)
	})
}

func Test_replaceExternalRulePlaceHolder(t *testing.T) {
	t.Run("replace annotations", func(t *testing.T) {
		promrule := mockOverlayPrometheusRule("id", "test", "test")
		rule := promrule.Spec.Groups[0].Rules[0]
		rule.Annotations[constants.RuleAnnotationsMessage] = "xxx {thresholdPercentage}% xxxx {severity}"
		rule.Labels[constants.RuleAnnotationsDescription] = "xxx {thresholdPercentage}% xxxx {severity}"
		rule.Annotations[constants.RuleAnnotationsSummary] = "xxx {threshold} xxxx"
		rule = replaceExternalRulePlaceHolder(rule)
		assert.Equal(t, rule.Annotations[constants.RuleAnnotationsMessage], "xxx 30% xxxx critical")
		assert.Equal(t, rule.Labels[constants.RuleAnnotationsDescription], "xxx 30% xxxx critical")
		assert.Equal(t, rule.Annotations[constants.RuleAnnotationsSummary], "xxx 0.3 xxxx")
	})
}

func Test_checkCanEdit(t *testing.T) {
	label := make(map[string]string)
	t.Run("none", func(t *testing.T) {
		label[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_NONE
		assert.Equal(t, constants.RULE_CANEDIT_NONE, checkCanEdit(label))
	})

	t.Run("status monitor empty", func(t *testing.T) {
		label[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_ALL
		assert.Equal(t, constants.RULE_CANEDIT_STATUS, checkCanEdit(label))
	})
	t.Run("status operator empty", func(t *testing.T) {
		label[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_ALL
		label[constants.RuleLabelMonitortarget] = "testtest"
		assert.Equal(t, constants.RULE_CANEDIT_STATUS, checkCanEdit(label))
	})
	t.Run("all", func(t *testing.T) {
		label[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_ALL
		label[constants.RuleLabelMonitortarget] = "testtest"
		label[constants.RuleLabelOperator] = "testtest"
		assert.Equal(t, constants.RULE_CANEDIT_ALL, checkCanEdit(label))
	})
}

func Test_getExternalAlertRules(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	t.Run("suc", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, err := getExternalAlertRules(context.TODO(), "id")
		assert.Nil(t, err)
		rule, ok := dict[fmt.Sprintf(constants.RULEID_TEMPLATE, "example", "1")]
		assert.True(t, ok)
		assert.Equal(t, rule.For, "10m")
		assert.Equal(t, len(promrule.Spec.Groups), 1)
	})
	t.Run("guery prometheusrule failed", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.UnrecognizedError, "unrecognized error")).Times(1)
		promrule, dict, err := getExternalAlertRules(context.TODO(), "id")
		assert.Error(t, err)
		assert.Nil(t, promrule)
		assert.Equal(t, len(dict), 0)
	})
	t.Run("rules empty", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		mockRule.Spec.Groups[0].Rules = nil
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, err := getExternalAlertRules(context.TODO(), "id")
		assert.Nil(t, err)
		assert.Equal(t, len(promrule.Spec.Groups), 0)
		assert.Equal(t, len(dict), 0)
	})

	t.Run("rule alert empty", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		mockRule.Spec.Groups[0].Rules[0].Alert = ""
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, err := getExternalAlertRules(context.TODO(), "id")
		assert.Nil(t, err)
		assert.Equal(t, len(promrule.Spec.Groups), 0)
		assert.Equal(t, len(dict), 0)
	})

	t.Run("external rules not exists", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		mockRule.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelVisibility] = ""
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, err := getExternalAlertRules(context.TODO(), "id")
		assert.Nil(t, err)
		assert.Equal(t, len(promrule.Spec.Groups), 0)
		assert.Equal(t, len(dict), 0)
	})

	t.Run("ruleID not exists", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		mockRule.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelID] = ""
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, err := getExternalAlertRules(context.TODO(), "id")
		assert.Nil(t, err)
		assert.Equal(t, len(promrule.Spec.Groups), 0)
		assert.Equal(t, len(dict), 0)
	})
}

func Test_getOverLayRules(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	t.Run("cluster not exists", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{},
			}, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "tenantID", "projectID", "applicationID", "bizTYpe")
		assert.Error(t, err)
		assert.Equal(t, overlayRuleExists, false)
		assert.Equal(t, len(dict), 0)
		assert.Nil(t, promrule)
	})

	t.Run("get cluster failed", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
						},
					},
				},
			}, errors.New(errors.ClusterNotFound, "unrecognized error")).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "tenantID", "projectID", "applicationID", "bizType")
		assert.Error(t, err)
		assert.Equal(t, overlayRuleExists, false)
		assert.Equal(t, len(dict), 0)
		assert.Nil(t, promrule)
	})

	t.Run("cluster base parse error", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "templateID",
							},
						},
					},
				},
			}, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "id", "id", "id", "bizType")
		assert.Error(t, err)
		assert.Equal(t, overlayRuleExists, false)
		assert.Equal(t, len(dict), 0)
		assert.Nil(t, promrule)
	})

	t.Run("rule templateID not Exists", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
						},
					},
				},
			}, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "id", "id", "id", "bizType")
		assert.Error(t, err)
		assert.Equal(t, overlayRuleExists, false)
		assert.Equal(t, len(dict), 0)
		assert.Nil(t, promrule)
	})

	t.Run("get template rule failed", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.PrometheusRuleNotFound, "unrecognized error")).Times(2)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "id", "id", "id", "id")
		assert.Error(t, err)
		assert.Equal(t, overlayRuleExists, false)
		assert.Equal(t, len(dict), 0)
		assert.Nil(t, promrule)
	})

	t.Run("get overlay rule failed", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.UnrecognizedError, "unrecognized error")).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "id", "id", "id", "id")
		assert.Error(t, err)
		assert.Equal(t, overlayRuleExists, true)
		assert.Equal(t, len(dict), 0)
		assert.Nil(t, promrule)
	})

	t.Run("suc", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.PrometheusRuleNotFound, "unrecognized error")).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "id",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "id", "id", "id", "id")
		assert.Nil(t, err)
		assert.Equal(t, overlayRuleExists, false)
		assert.Equal(t, len(dict), 1)
		assert.Equal(t, promrule.Name, fmt.Sprintf(constants.OVERLAY_NAME_TEMPATE, "id"))
	})

	t.Run("overlay exists merge suc", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(2)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "id",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		//promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		promrule, dict, overlayRuleExists, err := getOverLayRules(context.TODO(), "id", "id", "id", "id")
		assert.Nil(t, err)
		assert.Equal(t, overlayRuleExists, true)
		assert.Equal(t, len(dict), 1)
		assert.Equal(t, promrule.Name, fmt.Sprintf(constants.OVERLAY_NAME_TEMPATE, "id"))
	})
}

func Test_QueryAlertRulesByCloudUser(t *testing.T) {

	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)
	t.Run("getOverLayRules error", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
						},
					},
				},
			}, errors.New(errors.ClusterNotFound, "unrecognized error")).Times(1)
		//promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		resp, err := GetHandler().QueryAlertRulesByCloudUser(context.TODO(), message.QueryAlertRulesReq{TenantID: "test",
			ApplicationID: "test", ProjectID: "test"})
		assert.Error(t, err)
		assert.Nil(t, resp.AlertRuleList)
	})

	t.Run("query overlay suc", func(t *testing.T) {
		rules := []prometheusv1.Rule{
			{
				Alert: "HighRequestLatency1",
				Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.5"),
				For:   "30m",
				Labels: map[string]string{
					"severity":     "critical",
					"alerttype":    "customer",
					"ruleID":       "1",
					"userModified": "7",
					"visibility":   "external",
				},
				Annotations: map[string]string{
					"summary": "High request latency",
				},
			},
		}
		mockRuleOverlay := mockOverlayPrometheusRule("id", "test", "test")
		mockRuleOverlay.Spec.Groups[0].Rules = rules
		mockRule := mockOverlayPrometheusRule("id", "test", "test")

		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRuleOverlay, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		resp, err := GetHandler().QueryAlertRulesByCloudUser(context.TODO(), message.QueryAlertRulesReq{TenantID: "test",
			ApplicationID: "test", ProjectID: "test"})
		assert.Nil(t, err)
		assert.Equal(t, resp.AlertRuleList[0].Rules[0].DefaultFor, 10.0)
		assert.Equal(t, resp.AlertRuleList[0].Rules[0].DefaultThreshold, 0.3)
		assert.Equal(t, *resp.AlertRuleList[0].Rules[0].For, int64(30))
		assert.Equal(t, *resp.AlertRuleList[0].Rules[0].Threshold, 0.5)
	})

	t.Run("query suc", func(t *testing.T) {
		rules := []prometheusv1.Rule{
			{
				Alert: "HighRequestLatency4",
				Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.3"),
				For:   "10m",
				Labels: map[string]string{
					"severity":     "critical",
					"alerttype":    "customer",
					"ruleID":       "4",
					"userModified": "7",
					"visibility":   "external",
				},
				Annotations: map[string]string{
					"summary": "High request latency",
				},
			},
			{
				Alert: "HighRequestLatency3",
				Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.3"),
				For:   "10m",
				Labels: map[string]string{
					"severity":     "critical",
					"alerttype":    "customer",
					"ruleID":       "3",
					"userModified": "7",
					"visibility":   "external",
				},
				Annotations: map[string]string{
					"summary": "High request latency",
				},
			},
			{
				Alert: "HighRequestLatency2",
				Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.3"),
				For:   "10m",
				Labels: map[string]string{
					"severity":     "critical",
					"alerttype":    "customer",
					"ruleID":       "2",
					"userModified": "7",
					"visibility":   "external",
				},
				Annotations: map[string]string{
					"summary": "High request latency",
				},
			},
			{
				Alert: "HighRequestLatency1",
				Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.3"),
				For:   "10m",
				Labels: map[string]string{
					"severity":     "critical",
					"alerttype":    "customer",
					"ruleID":       "1",
					"userModified": "7",
					"visibility":   "external",
				},
				Annotations: map[string]string{
					"summary": "High request latency",
				},
			},
		}
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		mockRule.Spec.Groups[0].Rules = rules

		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.PrometheusRuleNotFound, "unrecognized error")).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		resp, err := GetHandler().QueryAlertRulesByCloudUser(context.TODO(), message.QueryAlertRulesReq{TenantID: "test",
			ApplicationID: "test", ProjectID: "test"})
		assert.Nil(t, err)
		assert.Equal(t, len(resp.AlertRuleList[0].Rules), 3)
	})

	t.Run("query cache", func(t *testing.T) {
		cacheConfig := structs.OverlayAlertRuleCacheConfig{
			ExpireTime:          3 * time.Second,
			MaxItems:            2,
			MaxDeletionPerCycle: 2,
			SleepTime:           1 * time.Second,
		}
		InitAlertRuleCache(cacheConfig)
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.PrometheusRuleNotFound, "unrecognized error")).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		resp, err := GetHandler().QueryAlertRulesByCloudUser(context.TODO(), message.QueryAlertRulesReq{TenantID: "test",
			ApplicationID: "test", ProjectID: "test"})
		assert.Nil(t, err)
		assert.Equal(t, len(resp.AlertRuleList[0].Rules), 1)

		respCache, errCache := GetHandler().QueryAlertRulesByCloudUser(context.TODO(), message.QueryAlertRulesReq{TenantID: "test",
			ApplicationID: "test", ProjectID: "test"})
		assert.Nil(t, errCache)
		assert.Equal(t, len(respCache.AlertRuleList[0].Rules), 1)
	})

}

func Test_UpdateAlertRulesByCloudUser(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	promeRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockPrometheusRuleReaderWriter(promeRuleRW)

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	t.Run("getOverLayRules error", func(t *testing.T) {
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
						},
					},
				},
			}, errors.New(errors.ClusterNotFound, "unrecognized error")).Times(1)
		//promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		_, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), message.UpdateAlertRulesByCloudUserReq{TenantID: "test",
			ApplicationID: "test", ProjectID: "test"})
		assert.Error(t, err)
	})

	t.Run("req rule not exists", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.PrometheusRuleNotFound, "unrecognized error")).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		req := message.UpdateAlertRulesByCloudUserReq{TenantID: "test", ProjectID: "test", ApplicationID: "test",
			AlertRules: map[string]structs.AlertRuleBase{
				"aaa": {
					AlertRuleID: "aaa",
					For:         utils.Int64Ptr(10),
				},
			}}
		resp, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), req)
		errs, ok := err.(errors.Error)

		assert.Equal(t, ok, true)
		assert.Equal(t, errs.Code(), errors.PrometheusRuleNotFound)
		assert.Equal(t, resp, message.UpdateAlertRulesByCloudUserResp{
			Data: map[string]structs.UpdateAlertRule{},
		})
	})

	t.Run("rule can not be edited", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promMock := mockRule
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_NONE
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&promMock, nil).Times(1)
		req := message.UpdateAlertRulesByCloudUserReq{
			TenantID: "test", ProjectID: "test", ApplicationID: "test",
			AlertRules: map[string]structs.AlertRuleBase{
				"example:1": {
					AlertRuleID: "example:1",
					For:         utils.Int64Ptr(0),
				},
			}}
		resp, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), req)
		errs, ok := err.(errors.Error)

		assert.Equal(t, ok, true)
		assert.Equal(t, errs.Code(), errors.PrometheusRuleCanNotBeEdit)
		assert.Equal(t, resp, message.UpdateAlertRulesByCloudUserResp{
			Data: map[string]structs.UpdateAlertRule{
				"example:1": {
					For:          "0s",
					OldFor:       "10m",
					Threshold:    0.3,
					OldThreshold: 0.3,
					AlertName:    "Test",
				},
			}})
	})
	t.Run("rule can not be edited for statusonly", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promMock := mockRule
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_STATUS
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&promMock, nil).Times(1)
		req := message.UpdateAlertRulesByCloudUserReq{
			TenantID: "test", ProjectID: "test", ApplicationID: "test",
			AlertRules: map[string]structs.AlertRuleBase{
				"example:1": {
					AlertRuleID: "example:1",
					For:         utils.Int64Ptr(10),
				},
			}}
		_, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), req)
		errs, ok := err.(errors.Error)

		assert.Equal(t, ok, true)
		assert.Equal(t, errs.Code(), errors.PrometheusRuleCanNotBeEdit)
	})

	t.Run("req rule status value not valid", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promMock := mockRule
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_ALL
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelOperator] = "test"
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelMonitortarget] = "test"
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&promMock, nil).Times(1)
		req := message.UpdateAlertRulesByCloudUserReq{
			TenantID: "test", ProjectID: "test", ApplicationID: "test",
			AlertRules: map[string]structs.AlertRuleBase{
				"example:1": {
					AlertRuleID: "example:1",
					For:         utils.Int64Ptr(10),
					Status:      "valid",
				},
			}}
		resp, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), req)
		errs, ok := err.(errors.Error)

		assert.Equal(t, ok, true)
		assert.Equal(t, errs.Code(), errors.PrometheusRuleStatusNotValid)
		assert.Equal(t, resp, message.UpdateAlertRulesByCloudUserResp{
			Data: map[string]structs.UpdateAlertRule{
				"example:1": {
					AlertName:    "Test",
					For:          "10m0s",
					OldFor:       "10m",
					Threshold:    0.3,
					OldThreshold: 0.3,
					Status:       "valid",
				},
			},
		})
	})

	t.Run("suc when overlay rule exists", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		mockRule.Spec.Groups[0].Rules[0].Annotations[constants.RuleAnnotationsDescription] = "Test {threshold}"
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, nil).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promMock := mockRule
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_ALL
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelOperator] = "test"
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelMonitortarget] = "test"
		threshold := 0.9
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&promMock, nil).Times(1)
		req := message.UpdateAlertRulesByCloudUserReq{
			TenantID: "test", ProjectID: "test", ApplicationID: "test",
			AlertRules: map[string]structs.AlertRuleBase{
				"example:1": {
					AlertRuleID: "example:1",
					For:         utils.Int64Ptr(10),
					Status:      "disabled",
					Threshold:   &threshold,
				},
			}}
		promeRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		resp, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), req)
		assert.Nil(t, err)
		assert.Equal(t, resp, message.UpdateAlertRulesByCloudUserResp{
			Data: map[string]structs.UpdateAlertRule{
				"example:1": {
					AlertName:    "Test 0.3",
					For:          "10m0s",
					OldFor:       "10m",
					OldThreshold: 0.3,
					Status:       "disabled",
					Threshold:    threshold,
				},
			},
		})
	})

	t.Run("suc when overlay rule not exists", func(t *testing.T) {
		mockRule := mockOverlayPrometheusRule("id", "test", "test")
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&mockRule, errors.New(errors.PrometheusRuleNotFound, "unrecognized error")).Times(1)
		clusterRW.EXPECT().QueryClusters(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return(
			&v1.ClusterList{
				Items: []v1.Cluster{
					{
						ObjectMeta: metav1.ObjectMeta{
							Name: "clsuter1",
							Annotations: map[string]string{
								constants.AnnotationShareMode: string(constants.SHARED),
							},
							Labels: map[string]string{
								constants.LabelTemplateID: "test",
								constants.LabelVendor:     "aws",
								constants.LabelRegion:     "us-west-2",
							},
						},
					},
				},
			}, nil).Times(1)
		promMock := mockRule
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelCanEdit] = constants.RULE_CANEDIT_ALL
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelOperator] = "test"
		promMock.Spec.Groups[0].Rules[0].Labels[constants.RuleLabelMonitortarget] = "test"
		promeRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), gomock.Any()).Return(&promMock, nil).Times(1)
		threshold := 0.9
		req := message.UpdateAlertRulesByCloudUserReq{
			TenantID: "test", ProjectID: "test", ApplicationID: "test",
			AlertRules: map[string]structs.AlertRuleBase{
				"example:1": {
					AlertRuleID: "example:1",
					For:         utils.Int64Ptr(10),
					Status:      "disabled",
					Threshold:   &threshold,
				},
			}}
		promeRuleRW.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		resp, err := GetHandler().UpdateAlertRulesByCloudUser(context.TODO(), req)
		assert.Nil(t, err)
		assert.Equal(t, resp, message.UpdateAlertRulesByCloudUserResp{
			Data: map[string]structs.UpdateAlertRule{
				"example:1": {
					AlertName:    "Test",
					For:          "10m0s",
					OldFor:       "10m",
					OldThreshold: 0.3,
					Status:       "disabled",
					Threshold:    threshold,
				},
			},
		})
	})
}

func Test_prepareParentCluster(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	t.Run("empty parent id", func(t *testing.T) {
		_, err := prepareParentCluster(context.TODO(), message.CreateClusterReq{})
		assert.NoError(t, err)
	})
	t.Run("not found", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), "notfound").Return(nil, errors.New(errors.ClusterNotFound, "")).Times(1)
		_, err := prepareParentCluster(context.TODO(), message.CreateClusterReq{
			ParentClusterID: "notfound",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})
	t.Run("not shared", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), "NotShared").Return(&v1.Cluster{}, nil).Times(1)
		_, err := prepareParentCluster(context.TODO(), message.CreateClusterReq{
			ParentClusterID: "NotShared",
			ClusterBaseInfo: structs.ClusterBaseInfo{
				ShareMode: constants.EXCLUSIVE,
			},
		})
		assert.NoError(t, err)
	})
	t.Run("supported", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), "supported").Return(&v1.Cluster{
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					SupportServerless: true,
				},
			},
		}, nil).Times(1)
		_, err := prepareParentCluster(context.TODO(), message.CreateClusterReq{
			ParentClusterID: "supported",
			ClusterBaseInfo: structs.ClusterBaseInfo{
				ShareMode: constants.SHARED,
			},
		})
		assert.NoError(t, err)
	})
	t.Run("update failed", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), "update").Return(&v1.Cluster{
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					SupportServerless: false,
				},
			},
		}, nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.ClusterStatusConflict, "")).Times(1)
		_, err := prepareParentCluster(context.TODO(), message.CreateClusterReq{
			ParentClusterID: "update",
			ClusterBaseInfo: structs.ClusterBaseInfo{
				ShareMode: constants.SHARED,
			},
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterStatusConflict, err.(errors.Error).Code())
	})
	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().GetCluster(gomock.Any(), "normal").Return(&v1.Cluster{
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					SupportServerless: false,
				},
			},
		}, nil).Times(1)
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := prepareParentCluster(context.TODO(), message.CreateClusterReq{
			ParentClusterID: "normal",
			ClusterBaseInfo: structs.ClusterBaseInfo{
				ShareMode: constants.SHARED,
			},
		})
		assert.NoError(t, err)
	})

}

func TestClusterHandler_ModifyAgentObserved(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	req1 := mockCreateReq()
	req1.ID = utils.GenerateID()

	cluster1, _ := BuildClusterResourceRequirements(context.TODO(), req1, "", nil)
	cluster1.Spec.Agent = &v1.Agent{}

	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()
	configs.Init(structs.ServiceInfo{
		MonitorSpec: map[string]structs.MonitorSpec{
			"default": {
				Cpu:                 "10m",
				Memory:              "200Mi",
				InitialStorage:      "40Gi",
				OverSell:            1,
				IsDefault:           true,
				MaxQueryConcurrency: pointer.Int32(10),
				MaxQuerySamples:     pointer.Int32(10000),
			},
		},
	})

	clusterRW.EXPECT().GetCluster(gomock.Any(), req1.ID).Return(cluster1, nil).AnyTimes()
	clusterRW.EXPECT().GetCluster(gomock.Any(), "not found").Return(nil, errors.New(errors.ClusterNotFound, "")).AnyTimes()

	t.Run("normal", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(nil).Times(1)
		_, err := GetHandler().ModifyAgentObserved(context.TODO(), message.ModifyAgentObservedReq{
			ID: req1.ID,
			Observed: structs.Observed{
				TiDBClusterTopology: &structs.ClusterTopologyType{
					TiDB: structs.TiDB{
						Replicas: 2,
					},
					TiKV: structs.TiKV{
						Replicas: 3,
					},
					PD: structs.PD{
						Replicas: 3,
					},
				},
				TiDBVersion: "8.5.1",
			},
		})
		assert.NoError(t, err)
		assert.Equal(t, 2, cluster1.Spec.Agent.Observed.TiDBClusterTopology.TiDB.Replicas)
		assert.Equal(t, "8.5.1", cluster1.Spec.Agent.Observed.TiDBVersion)
	})
	t.Run("invalid cluster id", func(t *testing.T) {
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     "not found",
			SpecID: "default",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.ClusterNotFound, err.(errors.Error).Code())
	})
	t.Run("update failed", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnrecognizedError, "")).Times(1)
		_, err := GetHandler().MonitorClusterSpec(context.TODO(), message.ModifyMonitorSpecReq{
			ID:     req1.ID,
			SpecID: "default",
		})
		assert.Error(t, err)
		assert.Equal(t, errors.UnrecognizedError, err.(errors.Error).Code())
	})
}

func TestClusterHandler_ModifyClusterMode(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	req1 := mockCreateReq()
	req1.ID = utils.GenerateID()
	t.Run("append prometheus failed", func(t *testing.T) {
		err := GetHandler().ModifyClusterMode(context.TODO(), &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster_name",
			}},
			message.CreateClusterReq{
				ClusterBaseInfo: structs.ClusterBaseInfo{
					ShareMode: constants.EXCLUSIVE,
				},
			})
		assert.Error(t, err)
		assert.Equal(t, errors.UnsupportedMonitorSpec, err.(errors.Error).Code())
	})
	t.Run("update failed", func(t *testing.T) {
		clusterRW.EXPECT().UpdateCluster(gomock.Any(), gomock.Any()).Return(errors.New(errors.UnsupportedMonitorSpec, "")).Times(1)
		err := GetHandler().ModifyClusterMode(context.TODO(), &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster_name",
				Annotations: map[string]string{
					constants.AnnotationShareMode: string(constants.EXCLUSIVE),
				},
				Labels: map[string]string{
					constants.LabelShareMode: string(constants.EXCLUSIVE),
				},
			}},
			message.CreateClusterReq{
				ClusterBaseInfo: structs.ClusterBaseInfo{
					ShareMode: constants.VIRTUAL,
				},
			})
		assert.Error(t, err)
		assert.Equal(t, errors.UnsupportedMonitorSpec, err.(errors.Error).Code())
	})
}

func mockPrometheusRuleForGetFull(id string, tenantID, projectID string) *prometheusv1.PrometheusRule {
	return &prometheusv1.PrometheusRule{
		ObjectMeta: metav1.ObjectMeta{
			Name:      id,
			Namespace: "default-" + tenantID,
			Labels: map[string]string{
				constants.LabelClusterID:     id,
				constants.LabelVendor:        "aws",
				constants.LabelRegion:        "us-east-2",
				constants.LabelTenantID:      tenantID,
				constants.LabelProjectID:     projectID,
				constants.LabelApplicationID: "default",
				constants.LabelBizType:       "default",
			},
			Annotations: map[string]string{
				constants.AnnotationNote: "note111",
			},
		},
		Spec: prometheusv1.PrometheusRuleSpec{
			Groups: []prometheusv1.RuleGroup{
				{
					Name:     "example",
					Interval: "5",
					Rules: []prometheusv1.Rule{
						{
							Alert: "HighRequestLatency",
							Expr:  intstr.FromString("request_latency_seconds:mean5m{job=\"myjob\"} > 0.5"),
							For:   "10m",
							Labels: map[string]string{
								"severity":  "critical",
								"alerttype": "customer",
							},
							Annotations: map[string]string{
								"summary": "High request latency",
							},
						},
					},
				},
			},
		},
	}
}

func TestClusterHandler_GetFullPrometheusRule(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	// set mock
	vmRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockVMRuleReaderWriter(vmRuleRW)

	handler := &ClusterHandler{}

	t.Run("normal - get prometheus rule successfully", func(t *testing.T) {
		clusterID := "test-cluster-123"
		mockRule := mockPrometheusRuleForGetFull(clusterID, "test-tenant", "test-project")

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(mockRule, nil).Times(1)

		resp, err := handler.GetFullPrometheusRule(context.TODO(), clusterID)
		assert.NoError(t, err)
		assert.NotNil(t, resp)
		assert.Equal(t, mockRule.Spec, resp.PrometheusRuleSpec)
		assert.Equal(t, len(mockRule.Spec.Groups), len(resp.Groups))
		assert.Equal(t, "example", resp.Groups[0].Name)
		assert.Equal(t, "5", resp.Groups[0].Interval)
		assert.Equal(t, 1, len(resp.Groups[0].Rules))
		assert.Equal(t, "HighRequestLatency", resp.Groups[0].Rules[0].Alert)
	})

	t.Run("error - prometheus rule not found", func(t *testing.T) {
		clusterID := "non-existent-cluster"

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(nil, errors.New(errors.PrometheusRuleNotFound, "PrometheusRule not found")).Times(1)

		resp, err := handler.GetFullPrometheusRule(context.TODO(), clusterID)
		assert.Error(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)

		errObj, ok := err.(errors.Error)
		assert.True(t, ok)
		assert.Equal(t, errors.PrometheusRuleNotFound, errObj.Code())
	})

	t.Run("error - get prometheus rule failed", func(t *testing.T) {
		clusterID := "error-cluster"

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(nil, errors.New(errors.UnrecognizedError, "Internal error")).Times(1)

		resp, err := handler.GetFullPrometheusRule(context.TODO(), clusterID)
		assert.Error(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)

		errObj, ok := err.(errors.Error)
		assert.True(t, ok)
		assert.Equal(t, errors.UnrecognizedError, errObj.Code())
	})
}

func TestClusterHandler_GetClusterExternal(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	clusterRW := mockresourcemanager.NewMockClusterReaderWriter(ctrl)
	resourcemanager.MockClusterReaderWriter(clusterRW)

	handler := &ClusterHandler{}

	t.Run("success - next gen cluster", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-123",
			ApplicationID: "app-456",
			BizType:       "production",
		}

		cluster := mockClusterForExternal("cluster-001", "tenant-123", "app-456", "production", true)
		clusters := &v1.ClusterList{Items: []v1.Cluster{*cluster}}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-123",
			"",
			"app-456",
			"production",
			gomock.Any(),
		).Return(clusters, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.NoError(t, err)
		assert.Equal(t, "cluster-001", resp.ID)
		assert.Equal(t, constants.AWS, resp.Vendor)
		assert.Equal(t, "us-east-1", resp.Region)
		assert.Equal(t, "tenant-123", resp.TenantID)
		assert.Equal(t, "app-456", resp.ApplicationID)
		assert.Equal(t, "production", resp.BizType)
		assert.NotNil(t, resp.Metrics)
		assert.NotNil(t, resp.SlowQuery)
		assert.NotNil(t, resp.KeyVisual)
		assert.Contains(t, resp.Metrics.OpenReadUri, "https://www.ds.")
		assert.Contains(t, resp.Metrics.OpenReadUri, "metrics")
	})

	t.Run("success - legacy cluster", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-456",
			ApplicationID: "app-789",
			BizType:       "development",
		}

		cluster := mockClusterForExternal("cluster-002", "tenant-456", "app-789", "development", false)
		clusters := &v1.ClusterList{Items: []v1.Cluster{*cluster}}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-456",
			"",
			"app-789",
			"development",
			gomock.Any(),
		).Return(clusters, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.NoError(t, err)
		assert.Equal(t, "cluster-002", resp.ID)
		assert.Equal(t, constants.GCP, resp.Vendor)
		assert.Equal(t, "us-central1", resp.Region)
		assert.Equal(t, "tenant-456", resp.TenantID)
		assert.Equal(t, "app-789", resp.ApplicationID)
		assert.Equal(t, "development", resp.BizType)
		assert.NotNil(t, resp.Metrics)
		assert.NotNil(t, resp.SlowQuery)
		assert.NotNil(t, resp.KeyVisual)
		// Legacy clusters now expose read-only endpoints for external access
		assert.Contains(t, resp.Metrics.OpenReadUri, "/external/metrics/")
		assert.Contains(t, resp.Metrics.OpenReadUri, "tenant/tenant-456")
		assert.Contains(t, resp.SlowQuery.OpenReadUri, "/external/slowquery/")
		assert.Contains(t, resp.KeyVisual.OpenReadUri, "/external/keyvisual/")
	})

	t.Run("error - query clusters failed", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-123",
			ApplicationID: "app-456",
			BizType:       "production",
		}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-123",
			"",
			"app-456",
			"production",
			gomock.Any(),
		).Return(nil, errors.New(errors.UnrecognizedError, "database error")).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.Error(t, err)
		assert.Equal(t, message.GetClusterExternalResp{}, resp)
	})

	t.Run("error - no clusters found", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-999",
			ApplicationID: "app-999",
			BizType:       "testing",
		}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-999",
			"",
			"app-999",
			"testing",
			gomock.Any(),
		).Return(&v1.ClusterList{Items: []v1.Cluster{}}, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.Error(t, err)
		assert.Equal(t, message.GetClusterExternalResp{}, resp)
		assert.True(t, errors.Is(err, errors.ClusterNotFound))
	})

	t.Run("error - multiple clusters found", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-multi",
			ApplicationID: "app-multi",
			BizType:       "staging",
		}

		clusters := &v1.ClusterList{
			Items: []v1.Cluster{
				*mockClusterForExternal("cluster-001", "tenant-multi", "app-multi", "staging", true),
				*mockClusterForExternal("cluster-002", "tenant-multi", "app-multi", "staging", false),
			},
		}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-multi",
			"",
			"app-multi",
			"staging",
			gomock.Any(),
		).Return(clusters, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.Error(t, err)
		assert.Equal(t, message.GetClusterExternalResp{}, resp)
		assert.True(t, errors.Is(err, errors.ParameterInvalid))
	})

	t.Run("error - cluster missing required labels", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-bad",
			ApplicationID: "app-bad",
			BizType:       "production",
		}

		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-bad",
				Labels: map[string]string{
					// Missing required labels
					constants.LabelTenantID: "tenant-bad",
				},
			},
		}
		clusters := &v1.ClusterList{Items: []v1.Cluster{*cluster}}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-bad",
			"",
			"app-bad",
			"production",
			gomock.Any(),
		).Return(clusters, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.Error(t, err)
		assert.Equal(t, message.GetClusterExternalResp{}, resp)
		assert.True(t, errors.Is(err, errors.ParameterInvalid))
	})

	t.Run("error - cluster with empty labels", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-empty",
			ApplicationID: "app-empty",
			BizType:       "production",
		}

		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "cluster-empty",
				Labels: map[string]string{},
			},
		}
		clusters := &v1.ClusterList{Items: []v1.Cluster{*cluster}}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-empty",
			"",
			"app-empty",
			"production",
			gomock.Any(),
		).Return(clusters, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.Error(t, err)
		assert.Equal(t, message.GetClusterExternalResp{}, resp)
		assert.True(t, errors.Is(err, errors.ParameterInvalid))
	})

	t.Run("normal - cluster with all external configs", func(t *testing.T) {
		req := message.GetClusterExternalReq{
			TenantID:      "tenant-full",
			ApplicationID: "app-full",
			BizType:       "tidbcloud",
		}

		cluster := mockClusterForExternal("cluster-full", "tenant-full", "app-full", "tidbcloud", true)
		clusters := &v1.ClusterList{Items: []v1.Cluster{*cluster}}

		clusterRW.EXPECT().QueryClusters(
			gomock.Any(),
			"tenant-full",
			"",
			"app-full",
			"tidbcloud",
			gomock.Any(),
		).Return(clusters, nil).Times(1)

		resp, err := handler.GetClusterExternal(context.Background(), req)
		assert.NoError(t, err)
		assert.Equal(t, "cluster-full", resp.ID)
		assert.Equal(t, constants.AWS, resp.Vendor)
		assert.Equal(t, "us-east-1", resp.Region)
		assert.Equal(t, "tenant-full", resp.TenantID)
		assert.Equal(t, "app-full", resp.ApplicationID)
		assert.Equal(t, "tidbcloud", resp.BizType)

		// Verify all external configurations are present
		assert.NotNil(t, resp.Metrics)
		assert.NotNil(t, resp.SlowQuery)
		assert.NotNil(t, resp.KeyVisual)

		assert.Equal(t,
			"https://o11y-ds.us-west-2.prod.aws.tidbcloud.com/cluster-full/metrics/api/v1/",
			resp.Metrics.OpenReadUri,
		)
		assert.Equal(t,
			"https://o11y-ds.us-west-2.prod.aws.tidbcloud.com/cluster-full/diagnosis/api/v1/",
			resp.SlowQuery.OpenReadUri,
		)
		assert.Equal(t,
			"https://o11y-ds.us-west-2.prod.aws.tidbcloud.com/cluster-full/diagnosis/api/v1/",
			resp.KeyVisual.OpenReadUri,
		)
		assert.Equal(t,
			"https://o11y-ds.us-west-2.prod.aws.tidbcloud.com/cluster-full/diagnosis/api/v1/",
			resp.SqlStatements.OpenReadUri,
		)
	})
}

// Helper function to create mock cluster for external tests
func mockClusterForExternal(id, tenantID, applicationID, bizType string, isNextGen bool) *v1.Cluster {
	labels := map[string]string{
		constants.LabelTenantID:      tenantID,
		constants.LabelApplicationID: applicationID,
		constants.LabelBizType:       bizType,
		constants.LabelVendor:        "aws",
		constants.LabelRegion:        "us-east-1",
	}

	if !isNextGen {
		labels[constants.LabelVendor] = "gcp"
		labels[constants.LabelRegion] = "us-central1"
	} else {
		labels[constants.LabelNextGen] = "true"
	}

	return &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:   id,
			Labels: labels,
		},
	}
}

func TestClusterHandler_UpdateFullPrometheusRule(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	vmRuleRW := mockresourcemanager.NewMockPrometheusRuleReaderWriter(ctrl)
	resourcemanager.MockVMRuleReaderWriter(vmRuleRW)

	handler := &ClusterHandler{}

	t.Run("success - update prometheus rule successfully", func(t *testing.T) {
		clusterID := "test-cluster-123"

		existingRule := mockPrometheusRuleForUpdate(clusterID, "test-tenant", "test-project")

		updateRules := message.OpenAlertRule{
			PrometheusRuleSpec: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name:     "updated-group",
						Interval: "10",
						Rules: []prometheusv1.Rule{
							{
								Alert: "UpdatedAlert",
								Expr:  intstr.FromString("updated_metric > 0.8"),
								For:   "5m",
								Labels: map[string]string{
									"severity": "warning",
								},
								Annotations: map[string]string{
									"summary": "Updated alert rule",
								},
							},
						},
					},
				},
			},
		}

		updatedRule := &prometheusv1.PrometheusRule{
			ObjectMeta: existingRule.ObjectMeta,
			Spec:       updateRules.PrometheusRuleSpec,
		}

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(existingRule, nil).Times(1)
		vmRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), updatedRule).Return(nil).Times(1)

		resp, err := handler.UpdateFullPrometheusRule(context.TODO(), clusterID, updateRules)
		assert.NoError(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp) // UpdateFullPrometheusRule should return an empty OpenAlertRule on success
	})

	t.Run("error - prometheus rule not found", func(t *testing.T) {
		clusterID := "non-existent-cluster"
		updateRules := message.OpenAlertRule{
			PrometheusRuleSpec: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name:     "test-group",
						Interval: "5",
						Rules:    []prometheusv1.Rule{},
					},
				},
			},
		}

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(nil, errors.New(errors.PrometheusRuleNotFound, "PrometheusRule not found")).Times(1)

		resp, err := handler.UpdateFullPrometheusRule(context.TODO(), clusterID, updateRules)
		assert.Error(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)

		errObj, ok := err.(errors.Error)
		assert.True(t, ok)
		assert.Equal(t, errors.PrometheusRuleNotFound, errObj.Code())
	})

	t.Run("error - get prometheus rule failed", func(t *testing.T) {
		clusterID := "error-cluster"
		updateRules := message.OpenAlertRule{
			PrometheusRuleSpec: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name:     "test-group",
						Interval: "5",
						Rules:    []prometheusv1.Rule{},
					},
				},
			},
		}

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(nil, errors.New(errors.UnrecognizedError, "Internal error")).Times(1)

		resp, err := handler.UpdateFullPrometheusRule(context.TODO(), clusterID, updateRules)
		assert.Error(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)

		errObj, ok := err.(errors.Error)
		assert.True(t, ok)
		assert.Equal(t, errors.UnrecognizedError, errObj.Code())
	})

	t.Run("error - update prometheus rule failed", func(t *testing.T) {
		clusterID := "update-error-cluster"

		existingRule := mockPrometheusRuleForUpdate(clusterID, "test-tenant", "test-project")

		updateRules := message.OpenAlertRule{
			PrometheusRuleSpec: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name:     "error-group",
						Interval: "15",
						Rules: []prometheusv1.Rule{
							{
								Alert: "ErrorAlert",
								Expr:  intstr.FromString("error_metric > 1"),
								For:   "2m",
								Labels: map[string]string{
									"severity": "critical",
								},
								Annotations: map[string]string{
									"summary": "Error occurred",
								},
							},
						},
					},
				},
			},
		}

		updatedRule := &prometheusv1.PrometheusRule{
			ObjectMeta: existingRule.ObjectMeta,
			Spec:       updateRules.PrometheusRuleSpec,
		}

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(existingRule, nil).Times(1)
		vmRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), updatedRule).Return(errors.New(errors.UnrecognizedError, "Update failed")).Times(1)

		resp, err := handler.UpdateFullPrometheusRule(context.TODO(), clusterID, updateRules)
		assert.Error(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)

		errObj, ok := err.(errors.Error)
		assert.True(t, ok)
		assert.Equal(t, errors.UnrecognizedError, errObj.Code())
	})

	t.Run("success - update with multiple rule groups", func(t *testing.T) {
		clusterID := "multi-group-cluster"

		existingRule := mockPrometheusRuleForUpdate(clusterID, "test-tenant", "test-project")

		updateRules := message.OpenAlertRule{
			PrometheusRuleSpec: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name:     "group1",
						Interval: "5",
						Rules: []prometheusv1.Rule{
							{
								Alert: "Alert1",
								Expr:  intstr.FromString("metric1 > 0.5"),
								For:   "1m",
								Labels: map[string]string{
									"severity": "warning",
								},
								Annotations: map[string]string{
									"summary": "First alert",
								},
							},
						},
					},
					{
						Name:     "group2",
						Interval: "10",
						Rules: []prometheusv1.Rule{
							{
								Alert: "Alert2",
								Expr:  intstr.FromString("metric2 > 0.8"),
								For:   "3m",
								Labels: map[string]string{
									"severity": "critical",
								},
								Annotations: map[string]string{
									"summary": "Second alert",
								},
							},
						},
					},
				},
			},
		}

		updatedRule := &prometheusv1.PrometheusRule{
			ObjectMeta: existingRule.ObjectMeta,
			Spec:       updateRules.PrometheusRuleSpec,
		}

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(existingRule, nil).Times(1)
		vmRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), updatedRule).Return(nil).Times(1)

		resp, err := handler.UpdateFullPrometheusRule(context.TODO(), clusterID, updateRules)
		assert.NoError(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)
	})

	t.Run("success - update with empty rule groups", func(t *testing.T) {
		clusterID := "empty-groups-cluster"

		existingRule := mockPrometheusRuleForUpdate(clusterID, "test-tenant", "test-project")

		updateRules := message.OpenAlertRule{
			PrometheusRuleSpec: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{},
			},
		}

		updatedRule := &prometheusv1.PrometheusRule{
			ObjectMeta: existingRule.ObjectMeta,
			Spec:       updateRules.PrometheusRuleSpec,
		}

		vmRuleRW.EXPECT().GetPrometheusRule(gomock.Any(), clusterID).Return(existingRule, nil).Times(1)
		vmRuleRW.EXPECT().UpdatePrometheusRule(gomock.Any(), updatedRule).Return(nil).Times(1)

		resp, err := handler.UpdateFullPrometheusRule(context.TODO(), clusterID, updateRules)
		assert.NoError(t, err)
		assert.Equal(t, message.OpenAlertRule{}, resp)
	})
}

// Helper function to create mock prometheus rule for update tests
func mockPrometheusRuleForUpdate(id string, tenantID, projectID string) *prometheusv1.PrometheusRule {
	return &prometheusv1.PrometheusRule{
		ObjectMeta: metav1.ObjectMeta{
			Name:      id,
			Namespace: "default-" + tenantID,
			Labels: map[string]string{
				constants.LabelClusterID:     id,
				constants.LabelVendor:        "aws",
				constants.LabelRegion:        "us-east-2",
				constants.LabelTenantID:      tenantID,
				constants.LabelProjectID:     projectID,
				constants.LabelApplicationID: "default",
				constants.LabelBizType:       "default",
			},
			Annotations: map[string]string{
				constants.AnnotationNote: "original note",
			},
		},
		Spec: prometheusv1.PrometheusRuleSpec{
			Groups: []prometheusv1.RuleGroup{
				{
					Name:     "original-group",
					Interval: "5",
					Rules: []prometheusv1.Rule{
						{
							Alert: "OriginalAlert",
							Expr:  intstr.FromString("original_metric > 0.5"),
							For:   "5m",
							Labels: map[string]string{
								"severity":  "warning",
								"alerttype": "customer",
							},
							Annotations: map[string]string{
								"summary": "Original alert rule",
							},
						},
					},
				},
			},
		},
	}
}

// TestParseInternalReadUri tests the parseInternalReadUri function
func TestParseInternalReadUri(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	// Setup mock service info
	configs.Init(structs.ServiceInfo{
		ServerName:        constants.ObservabilityRegionalServer,
		LogicalRegionName: "us-east-1",
		Vendor:            "aws",
		Region:            "us-east-1",
		Env:               constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName:     "www.ds.aws.observability-dev.pingcap.cloud",
				GlobalDataServerHostname: "o11y-ds.us-west-2.nightly.shared.aws.tidbcloud.com",
			},
		},
	})

	t.Run("next-gen cluster with parent cluster", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "child-cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen:         "true",
					constants.LabelParentClusterID: "parent-cluster-456",
				},
			},
		}

		result := parseInternalReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/v2/metrics/parent-cluster-456/api/v1/"
		assert.Equal(t, expected, result)
	})

	t.Run("next-gen cluster without parent cluster", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen: "true",
				},
			},
		}

		result := parseInternalReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/v2/metrics/cluster-123/api/v1/"
		assert.Equal(t, expected, result)
	})

	t.Run("non-next-gen cluster", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name:   "cluster-123",
				Labels: map[string]string{
					// No next-gen label
				},
			},
		}

		result := parseInternalReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/metrics/cluster-123/api/v1/"
		assert.Equal(t, expected, result)
	})

	t.Run("cluster with next-gen label false", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen: "false",
				},
			},
		}

		result := parseInternalReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/metrics/cluster-123/api/v1/"
		assert.Equal(t, expected, result)
	})

	// byoc cluster uses O11yGlobalDSHostName for internal read uri
	t.Run("byoc cluster uses O11yGlobalDSHostName for internal read uri", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelBizType: "byoc",
				},
			},
		}

		result := parseInternalReadUri(context.Background(), cluster)
		expected := "https://www.ds.aws.observability-dev.pingcap.cloud/internal/cluster-123/metrics/api/v1/"
		assert.Equal(t, expected, result)
	})

	// O11yGlobalDSHostName is the single source for internal read URI; InternalGlobalDSHostName is not consulted
	t.Run("byoc cluster uses O11yGlobalDSHostName directly for internal read uri", func(t *testing.T) {
		original := configs.GetServiceInfo()
		defer func() { configs.Init(original) }()
		configs.Init(structs.ServiceInfo{
			ServerName: constants.ObservabilityRegionalServer,
			Vendor:     "aws",
			Region:     "us-east-1",
			Env:        constants.EnvDev,
			BizTypeConfigs: map[string]structs.BizTypeConfig{
				constants.DefaultByocBizType: {
					O11yGlobalDSHostName:     "www.ds.observability-dev.pingcap.cloud",
					GlobalDataServerHostname: "o11y-ds.us-west-2.nightly.shared.aws.tidbcloud.com",
				},
			},
		})

		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelBizType: "byoc",
				},
			},
		}

		result := parseInternalReadUri(context.Background(), cluster)
		expected := "https://www.ds.observability-dev.pingcap.cloud/internal/cluster-123/metrics/api/v1/"
		assert.Equal(t, expected, result)
	})
}

func TestBuildRegionalLogQueryURI(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	configs.Init(structs.ServiceInfo{
		ServerName:        constants.ObservabilityRegionalServer,
		LogicalRegionName: "us-east-1",
		Vendor:            "aws",
		Region:            "us-east-1",
		Env:               constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName: "www.ds.aws.observability-dev.pingcap.cloud",
			},
		},
	})

	t.Run("non-byoc uses ds host with public loki path", func(t *testing.T) {
		result := buildRegionalLogQueryURI("tidbcloud")
		assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/loki/self-monitoring", result)
	})

	t.Run("byoc uses resolved ds host with internal loki path", func(t *testing.T) {
		result := buildRegionalLogQueryURI(constants.DefaultByocBizType)
		assert.Equal(t, "https://www.ds.aws.observability-dev.pingcap.cloud/internal/loki/self-monitoring", result)
	})
}

func TestBuildRegionalMultitenantQueryURI(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	configs.Init(structs.ServiceInfo{
		ServerName:        constants.ObservabilityRegionalServer,
		LogicalRegionName: "us-east-1",
		Vendor:            "aws",
		Region:            "us-east-1",
		Env:               constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName: "www.ds.aws.observability-dev.pingcap.cloud",
			},
		},
	})

	t.Run("returns empty when vm account is missing", func(t *testing.T) {
		assert.Empty(t, buildRegionalMultitenantQueryURI("tidbcloud", 0))
	})

	t.Run("non-byoc uses regional ds host", func(t *testing.T) {
		result := buildRegionalMultitenantQueryURI("tidbcloud", 1212)
		assert.Equal(
			t,
			"https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/v2/metrics/multitenant/api/v1/?extra_filters%5B%5D=%7Bvm_account_id%3D%221212%22%7D",
			result,
		)
	})

	t.Run("byoc uses resolved ds host", func(t *testing.T) {
		result := buildRegionalMultitenantQueryURI(constants.DefaultByocBizType, 1212)
		assert.Equal(
			t,
			"https://www.ds.aws.observability-dev.pingcap.cloud/internal/multitenant/metrics/api/v1/?extra_filters%5B%5D=%7Bvm_account_id%3D%221212%22%7D",
			result,
		)
	})
}

func TestBuildRegionalDiagnosisQueryURI(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	configs.Init(structs.ServiceInfo{
		ServerName:        constants.ObservabilityRegionalServer,
		LogicalRegionName: "us-east-1",
		Vendor:            "aws",
		Region:            "us-east-1",
		Env:               constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName: "www.ds.aws.observability-dev.pingcap.cloud",
			},
		},
	})

	t.Run("non-byoc uses regional ds host", func(t *testing.T) {
		result := buildRegionalDiagnosisQueryURI("tidbcloud")
		assert.Equal(t, "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/internal/diagnosis/api/v1/", result)
	})

	t.Run("byoc uses resolved ds host", func(t *testing.T) {
		result := buildRegionalDiagnosisQueryURI(constants.DefaultByocBizType)
		assert.Equal(t, "https://www.ds.aws.observability-dev.pingcap.cloud/internal/diagnosis/api/v1/", result)
	})
}

func TestBuildStorageInfoAliCloud(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	configs.Init(structs.ServiceInfo{
		ServerName: constants.ObservabilityRegionalServer,
		Vendor:     string(constants.AliCloud),
		Region:     "ap-southeast-1",
		Env:        constants.EnvDev,
		AccountInfo: structs.AccountInfoConfig{
			Account: "ali-account-1",
		},
	})

	arnTemplate, storageAccount := getStorageLocationMetadata()
	info := buildStorageInfo(
		structs.ClusterBaseInfo{
			TenantID:      "tenant-ali",
			ProjectID:     "project-ali",
			ApplicationID: "app-ali",
		},
		"ali-bucket",
		storageAccount,
		arnTemplate,
		constants.SLOWQUERYS,
	)

	assert.Equal(t, "oss://ali-bucket/0/tenant-ali/project-ali/app-ali/logs/slowlog/", info.DataARN)
	assert.Equal(t, "ali-account-1", info.StorageAccount)
	assert.Equal(t, "0/tenant-ali/project-ali/app-ali/logs/slowlog/", info.ObjectPath)

	t.Run("serverless object path includes bucket prefix", func(t *testing.T) {
		serverlessInfo := buildStorageInfo(
			structs.ClusterBaseInfo{
				TenantID:      "tenant-ali",
				ProjectID:     "project-ali",
				ApplicationID: "app-ali",
				BizType:       "prod-devtier",
				CommonReqInfo: structs.CommonReqInfo{
					Vendor: constants.AliCloud,
				},
			},
			"ali-bucket",
			storageAccount,
			arnTemplate,
			constants.SQLSTATEMENTS,
		)

		assert.Equal(t, "ali-bucket/0/prod-devtier/serverless/tenant-ali/project-ali/app-ali/logs/statement/", serverlessInfo.ObjectPath)
	})
}

func TestBuildStorageObjectPath(t *testing.T) {
	t.Run("dedicated uses tenant project application category", func(t *testing.T) {
		path := buildStorageObjectPath(structs.ClusterBaseInfo{
			TenantID:           "tenant-a",
			ProjectID:          "project-a",
			ApplicationID:      "app-a",
			BizType:            "tidbcloud",
			DataPropertyRights: constants.ServiceProvider,
		}, "bucket-a", constants.SLOWQUERYS)
		assert.Equal(t, "0/tenant-a/project-a/app-a/logs/slowlog/", path)
	})

	t.Run("serverless non-alicloud keeps existing path format", func(t *testing.T) {
		path := buildStorageObjectPath(structs.ClusterBaseInfo{
			TenantID:      "tenant-s",
			ProjectID:     "project-s",
			ApplicationID: "app-s",
			BizType:       "prod-devtier",
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AWS,
			},
		}, "aws-bucket", constants.SQLSTATEMENTS)
		assert.Equal(t, "0/prod-devtier/serverless/tenant-s/project-s/app-s/logs/statement/", path)
	})

	t.Run("serverless alicloud adds bucket prefix", func(t *testing.T) {
		path := buildStorageObjectPath(structs.ClusterBaseInfo{
			TenantID:      "tenant-ali",
			ProjectID:     "project-ali",
			ApplicationID: "app-ali",
			BizType:       "prod-devtier",
			CommonReqInfo: structs.CommonReqInfo{
				Vendor: constants.AliCloud,
			},
		}, "ali-bucket", constants.SQLSTATEMENTS)
		assert.Equal(t, "ali-bucket/0/prod-devtier/serverless/tenant-ali/project-ali/app-ali/logs/statement/", path)
	})

	t.Run("premium shared uses next-gen path", func(t *testing.T) {
		path := buildStorageObjectPath(structs.ClusterBaseInfo{
			ApplicationID: "app-p",
			ShareMode:     constants.SHARED,
			BizType:       "dev-tidbcloud-nextgen",
		}, "bucket-p", constants.SLOWQUERYS)
		assert.Equal(t, "0/next-gen/app-p/logs/slowlog/", path)
	})

	t.Run("premium non-shared uses next-gen path", func(t *testing.T) {
		path := buildStorageObjectPath(structs.ClusterBaseInfo{
			ApplicationID: "app-p2",
			ShareMode:     constants.EXCLUSIVE,
			BizType:       constants.DefaultByocBizType,
		}, "bucket-p2", constants.SQLSTATEMENTS)
		assert.Equal(t, "0/next-gen/app-p2/logs/statementlog/", path)
	})
}

// TestParseOpenReadReadUri tests the parseOpenReadReadUri function
func TestParseOpenReadReadUri(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	// Setup mock service info
	configs.Init(structs.ServiceInfo{
		ServerName: constants.ObservabilityRegionalServer,
		Vendor:     "aws",
		Region:     "us-east-1",
		Env:        constants.EnvDev,
	})

	t.Run("next-gen cluster", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen:       "true",
					constants.LabelBizType:       "tidbcloud",
					constants.LabelTenantID:      "tenant-123",
					constants.LabelApplicationID: "app-456",
				},
			},
		}

		result := parseO11yOpenReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/external/metrics/tidbcloud/tenant/tenant-123/application/app-456/api/v1/"
		assert.Equal(t, expected, result)
	})

	t.Run("non-next-gen cluster", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelBizType:       "tidbcloud",
					constants.LabelTenantID:      "tenant-123",
					constants.LabelProjectID:     "project-789",
					constants.LabelApplicationID: "app-456",
				},
			},
		}

		result := parseO11yOpenReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/external/metrics/tidbcloud/tenant/tenant-123/project/project-789/application/app-456/api/v1/"
		assert.Equal(t, expected, result)
	})

	t.Run("next-gen cluster with missing labels", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen: "true",
					// Missing some required labels
					constants.LabelBizType: "tidbcloud",
				},
			},
		}

		result := parseO11yOpenReadUri(context.Background(), cluster)
		expected := "https://www.ds.us-east-1.aws.observability-dev.pingcap.cloud/external/metrics/tidbcloud/tenant//application//api/v1/"
		assert.Equal(t, expected, result)
	})
}

func TestParseO11yOpenReadUriUsesGlobalHostname(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	configs.Init(structs.ServiceInfo{
		ServerName: constants.ObservabilityRegionalServer,
		Vendor:     "aws",
		Region:     "us-east-1",
		Env:        constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName:     "www.ds.aws.observability-dev.pingcap.cloud",
				GlobalDataServerHostname: "o11y-ds.us-west-2.nightly.shared.aws.tidbcloud.com",
			},
		},
	})

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name: "cluster-123",
			Labels: map[string]string{
				constants.LabelBizType: "byoc",
			},
		},
	}

	result := parseO11yOpenReadUri(context.Background(), cluster)
	expected := "https://www.ds.aws.observability-dev.pingcap.cloud/cluster-123/metrics/api/v1/"
	assert.Equal(t, expected, result)
}

// TestParseO11yOpenReadUriByocUsesO11yGlobalDSHostName tests that parseO11yOpenReadUri uses
// O11yGlobalDSHostName when configured for BYOC clusters.
func TestParseO11yOpenReadUriByocUsesO11yGlobalDSHostName(t *testing.T) {
	originalService := configs.GetServiceInfo()
	defer func() {
		configs.Init(originalService)
	}()

	configs.Init(structs.ServiceInfo{
		ServerName: constants.ObservabilityRegionalServer,
		Vendor:     "aws",
		Region:     "us-east-1",
		Env:        constants.EnvDev,
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			constants.DefaultByocBizType: {
				O11yGlobalDSHostName:     "www.ds.observability-dev.pingcap.cloud",
				GlobalDataServerHostname: "o11y-ds.us-west-2.nightly.shared.aws.tidbcloud.com",
			},
		},
	})

	cluster := &v1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name: "cluster-123",
			Labels: map[string]string{
				constants.LabelBizType: "byoc",
			},
		},
	}

	result := parseO11yOpenReadUri(context.Background(), cluster)
	expected := "https://www.ds.observability-dev.pingcap.cloud/cluster-123/metrics/api/v1/"
	assert.Equal(t, expected, result)
}

// TestArchiveNextGen tests the archive function with next-gen cluster
func TestArchiveNextGen(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	archivedRW := mockresourcemanager.NewMockArchivedClusterReaderWriter(ctrl)
	resourcemanager.MockArchivedClusterReaderWriter(archivedRW)

	baseInfo := structs.ClusterBaseInfo{
		CommonReqInfo: structs.CommonReqInfo{
			Vendor: constants.AWS,
			Region: "us-east-1",
		},
		TenantID:      "tenant-123",
		ProjectID:     "project-456",
		ApplicationID: "app-789",
		BizType:       "tidbcloud",
		ShareMode:     constants.SHARED,
	}

	t.Run("next-gen shared cluster archive success", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen:         "true",
					constants.LabelShareMode:       string(constants.SHARED),
					constants.LabelProfilingID:     "profiling-project-123",
					constants.LabelSlowQueryID:     "slowquery-project-456",
					constants.LabelKeyVisualizerID: "keyvis-project-789",
					constants.LabelTopSQLID:        "topsql-project-012",
				},
			},
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					AccountID: 12345,
					ProjectID: 67890,
				},
			},
		}

		// Expected archived cluster info with all categories
		expectedArchive := structs.ArchivedClusterInfo{
			ClusterBaseInfo: baseInfo,
			VMAccountID:     "12345",
			VMProjectID:     "67890",
			StorageInfo: []structs.StorageInfo{
				{
					Category: constants.LOGS,
				},
				{
					Category: constants.PROFILING,
				},
				{
					Category: constants.SLOWQUERYS,
				},
				{
					Category: constants.SQLSTATEMENTS,
				},
			},
		}

		archivedRW.EXPECT().Archive(
			gomock.Any(),
			gomock.Any(),
			gomock.Any(),
			gomock.Eq(expectedArchive),
		).Return(nil).Times(1)

		err := archive(context.Background(), cluster, baseInfo)
		assert.NoError(t, err)
	})

	t.Run("exclusive cluster - no archive", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen:   "true",
					constants.LabelShareMode: string(constants.EXCLUSIVE),
				},
			},
		}

		// No expectations set for archivedRW since exclusive clusters should not be archived
		err := archive(context.Background(), cluster, baseInfo)
		assert.NoError(t, err)
	})

	t.Run("non-next-gen shared cluster", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelShareMode: string(constants.SHARED),
					// No next-gen label
				},
			},
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					AccountID: 12345,
					ProjectID: 67890,
				},
			},
		}

		// Expected archived cluster info without storage info (non-next-gen)
		expectedArchive := structs.ArchivedClusterInfo{
			ClusterBaseInfo: baseInfo,
			VMAccountID:     "12345",
			VMProjectID:     "67890",
			StorageInfo:     nil, // Non-next-gen clusters don't have storage info
		}

		archivedRW.EXPECT().Archive(
			gomock.Any(),
			gomock.Any(),
			gomock.Any(),
			gomock.Eq(expectedArchive),
		).Return(nil).Times(1)

		err := archive(context.Background(), cluster, baseInfo)
		assert.NoError(t, err)
	})

	t.Run("archive fails", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name: "cluster-123",
				Labels: map[string]string{
					constants.LabelNextGen:   "true",
					constants.LabelShareMode: string(constants.SHARED),
				},
			},
			Spec: v1.ClusterSpec{
				VictoriaAccount: v1.VictoriaAccount{
					AccountID: 12345,
					ProjectID: 67890,
				},
			},
		}

		archivedRW.EXPECT().Archive(
			gomock.Any(),
			gomock.Any(),
			gomock.Any(),
			gomock.Any(),
		).Return(errors.New(errors.UnrecognizedError, "Archive failed")).Times(1)

		err := archive(context.Background(), cluster, baseInfo)
		assert.Error(t, err)

		errObj, ok := err.(errors.Error)
		assert.True(t, ok)
		assert.Equal(t, errors.UnrecognizedError, errObj.Code())
	})
}

// TestIsNextGenCluster tests the isNextGenCluster helper function
func TestIsNextGenCluster(t *testing.T) {
	t.Run("cluster with next-gen label true", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelNextGen: "true",
				},
			},
		}

		result := isNextGenCluster(cluster)
		assert.True(t, result)
	})

	t.Run("cluster with next-gen label false", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelNextGen: "false",
				},
			},
		}

		result := isNextGenCluster(cluster)
		assert.False(t, result)
	})

	t.Run("cluster without next-gen label", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					"other-label": "value",
				},
			},
		}

		result := isNextGenCluster(cluster)
		assert.False(t, result)
	})

	t.Run("cluster with empty labels", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{},
			},
		}

		result := isNextGenCluster(cluster)
		assert.False(t, result)
	})

	t.Run("cluster with nil labels", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: nil,
			},
		}

		result := isNextGenCluster(cluster)
		assert.False(t, result)
	})

	t.Run("cluster with next-gen label with different value", func(t *testing.T) {
		cluster := &v1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Labels: map[string]string{
					constants.LabelNextGen: "yes",
				},
			},
		}

		result := isNextGenCluster(cluster)
		assert.False(t, result)
	})
}
func TestClusterHandler_ApplyLogAlertRules(t *testing.T) {
	ctrl := gomock.NewController(t)
	defer ctrl.Finish()

	configMapRW := mockresourcemanager.NewMockConfigMapReaderWriter(ctrl)
	resourcemanager.MockConfigMapReaderWriter(configMapRW)

	t.Run("success", func(t *testing.T) {
		configMapRW.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any()).Return(nil).Times(1)

		req := message.ApplyLogAlertRulesReq{
			RulesConfig: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name: "log-alerts",
						Rules: []prometheusv1.Rule{
							{
								Alert: "HighErrorRate",
								Expr:  intstr.FromString(`rate({job="app"} |= "error" [5m]) > 0.1`),
								For:   "5m",
								Labels: map[string]string{
									"severity": "warning",
								},
								Annotations: map[string]string{
									"summary": "High error rate detected",
								},
							},
						},
					},
				},
			},
		}

		_, err := GetHandler().ApplyLogAlertRules(context.TODO(), req)
		assert.NoError(t, err)
	})

	t.Run("empty rules", func(t *testing.T) {
		req := message.ApplyLogAlertRulesReq{
			RulesConfig: prometheusv1.PrometheusRuleSpec{},
		}

		_, err := GetHandler().ApplyLogAlertRules(context.TODO(), req)
		assert.Error(t, err)
		assert.Contains(t, err.Error(), "log alert rule not allow empty")
	})

	t.Run("should configmap create error", func(t *testing.T) {
		configMapRW.EXPECT().CreateOrUpdate(gomock.Any(), gomock.Any()).Return(fmt.Errorf("create failed")).Times(1)

		req := message.ApplyLogAlertRulesReq{
			RulesConfig: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name: "log-alerts",
						Rules: []prometheusv1.Rule{
							{
								Alert: "TestAlert",
								Expr:  intstr.FromString(`{job="test"}`),
							},
						},
					},
				},
			},
		}

		_, err := GetHandler().ApplyLogAlertRules(context.TODO(), req)
		assert.Error(t, err)
		assert.Contains(t, err.Error(), "failed to create or update ConfigMap")
	})

	t.Run("should fail: duplicate rule groups", func(t *testing.T) {

		req := message.ApplyLogAlertRulesReq{
			RulesConfig: prometheusv1.PrometheusRuleSpec{
				Groups: []prometheusv1.RuleGroup{
					{
						Name: "duplicate-group",
						Rules: []prometheusv1.Rule{
							{Alert: "Alert1", Expr: intstr.FromString(`{job="test1"}`)},
						},
					},
					{
						Name: "duplicate-group", // Same name should be deduplicated
						Rules: []prometheusv1.Rule{
							{Alert: "Alert2", Expr: intstr.FromString(`{job="test2"}`)},
						},
					},
					{
						Name: "unique-group",
						Rules: []prometheusv1.Rule{
							{Alert: "Alert3", Expr: intstr.FromString(`{job="test3"}`)},
						},
					},
				},
			},
		}

		_, err := GetHandler().ApplyLogAlertRules(context.TODO(), req)
		assert.Error(t, err)
	})
}