package cluster

import (
	"context"
	"errors"
	"fmt"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	vmv1 "github.com/VictoriaMetrics/operator/api/operator/v1beta1"
	. "github.com/onsi/gomega"
	prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
	observabilitypingcapcomv1 "github.com/tidbcloud/observability/api/regionalserver/resourceworker/v1"
	"github.com/tidbcloud/observability/pkg/common/configs"
	"github.com/tidbcloud/observability/pkg/common/constants"
	"github.com/tidbcloud/observability/pkg/common/metrics"
	"github.com/tidbcloud/observability/pkg/common/structs"
	"github.com/tidbcloud/observability/pkg/common/utils"
	"github.com/tidbcloud/observability/pkg/resourceworker/controllers/fake"
	mock_manager "github.com/tidbcloud/observability/pkg/resourceworker/controllers/fake/manager"
	"github.com/tidbcloud/observability/pkg/resourceworker/controllers/reconciler"
	"go.uber.org/mock/gomock"
	k8sapiv1 "k8s.io/api/core/v1"
	k8snetworkv1 "k8s.io/api/networking/v1"
	pkgerrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/pointer"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	ctrlfake "sigs.k8s.io/controller-runtime/pkg/client/fake"
	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config"
)

const (
	bizType           = "tidbcloud"
	bizTypeServerless = "serverless"
)

type args struct {
	cli         func(g Gomega) client.Client
	resource    func() client.Object
	serviceInfo *structs.ServiceInfo
}

type want struct {
	result  interface{}
	err     error
	paniced bool
}
type caseType map[string]struct {
	args args
	want want
}

var (
	notfound = func(gv schema.GroupVersionKind, name string) *pkgerrors.StatusError {
		gp := schema.GroupResource{
			Group:    gv.Group,
			Resource: gv.Kind,
		}
		return pkgerrors.NewNotFound(gp, name)
	}
)

func initConfig() {
	configs.Init(structs.ServiceInfo{
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			"test1": {
				BizType:                    "test1",
				ExternalApiAuthUrlTemplate: "https://test/api/v1/users/me",
				AgentManagerARN:            "aaa",
				MonitorPromRuleTemplate:    "rule-template",
				MonitorReplicas:            1,
				MonitorVersion:             "v2.33.5",
				MonitorRetention:           "7d",
			},
		},
	})
}

func TestConnect(t *testing.T) {
	_, err := clusterConnector{}.Connect(context.TODO(), nil)
	assert.NoError(t, err)
}

func TestClusterControllerBuildStruct(t *testing.T) {
	g := NewGomegaWithT(t)
	ctl := gomock.NewController(t)
	defer ctl.Finish()
	manager := mock_manager.NewMockManager(ctl)
	errBoom := errors.New("boom")
	mockCli := &fake.MockClient{
		MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
			return errBoom
		}),
	}
	scheme := runtime.NewScheme()
	log := ctrl.Log.WithName("controllers").WithName("IAM")
	rec := record.NewFakeRecorder(1)
	azconfig := make(map[string]structs.AvailabilityZoneConfig)
	serviceinfo := structs.ServiceInfo{
		AZConfig: azconfig,
	}
	manager.EXPECT().GetClient().AnyTimes().Return(mockCli)
	manager.EXPECT().GetEventRecorderFor(gomock.Any()).Return(rec)
	iam := NewClusterReconciler(manager, log, serviceinfo)
	g.Expect(iam.Log).Should(Equal(log))
	g.Expect(iam.reconciler.Client).Should(Equal(mockCli))
	utilruntime.Must(observabilitypingcapcomv1.AddToScheme(scheme))
	// manager.EXPECT().SetFields(gomock.Any()).AnyTimes().Return(nil)
	manager.EXPECT().Add(gomock.Any()).AnyTimes().Return(nil)
	manager.EXPECT().GetControllerOptions().Return(ctrlconfig.Controller{}).AnyTimes()
	manager.EXPECT().GetScheme().Return(scheme)
	manager.EXPECT().GetCache().Return(nil)
	manager.EXPECT().GetLogger().Return(log)
	g.Expect(iam.SetupWithManager(manager)).Should(Succeed())
	g.Expect(iam.Reconcile(context.TODO(), ctrl.Request{})).Error().Should(HaveOccurred())
}

func TestClusterControllerObserve(t *testing.T) {
	metrics.SetupMetrics()
	errBoom := errors.New("boom")
	noutfound := func(gv schema.GroupVersionKind, name string) *pkgerrors.StatusError {
		gp := schema.GroupResource{
			Group:    gv.Group,
			Resource: gv.Kind,
		}
		return pkgerrors.NewNotFound(gp, name)
	}
	_, errtimeprase := time.Parse(time.RFC3339, "test1")
	NowTime := time.Now().UTC()
	cases := caseType{
		"ObserveErrorResourceType": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{}
				},
				resource: func() client.Object {
					return &observabilitypingcapcomv1.S3{}
				},
			},
			want: want{
				result: reconciler.ExternalObservation{Droped: true},
				err:    errors.New(errNotCluster),
			},
		},
		"ObserveFinalStatus": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Status.Phase = constants.CREATION_FAILED
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceUpToDate: true, ResourceExists: true},
			},
		},
		"ObserveCompositeResourceError": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{},
				err:    errBoom,
			},
		},
		"ObserveCompositeResourceNotExist": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.MetricsToObjectStorage = true
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: false, PartResourceExists: false},
			},
		},
		"ObserveCompositeResourcePartNotExist": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if _, ok := obj.(*k8sapiv1.Service); ok {
								return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: false, PartResourceExists: true},
			},
		},
		"ObserveCompositeResourcePartNotExist-Shared": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if _, ok := obj.(*k8snetworkv1.Ingress); ok {
								if key.Name == svcPrefix+"test1" {
									return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
								}
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: false, PartResourceExists: true},
			},
		},
		"ObserveCompositeResourceAllExist": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, PartResourceExists: false},
			},
		},
		"ObserveCompositeResourceAllExist-Shared": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, PartResourceExists: false},
			},
		},
		"ObserveServerlessCluster-Exist": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Labels = map[string]string{
						constants.LabelBizType: bizType,
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.VictoriaAccount.ProjectID = 100
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false},
			},
		},
		"ObserveStatusCreatingWaitPrometheus": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionFalse,
								})
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.CREATING
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pollInterval},
			},
		},
		"observeCompositeResource--ServerlessCluster-ServerlessIngressExists": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							// Mock all resources exist, including serverless ingress
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Labels = map[string]string{
						constants.LabelBizType: bizType,
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.VictoriaAccount.SupportServerless = true // Enable serverless support
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, PartResourceExists: false},
			},
		},
		"observeCompositeResource--ServerlessCluster-ServerlessIngressNotFound": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							// Return not found specifically for serverless ingress
							if _, ok := obj.(*k8snetworkv1.Ingress); ok {
								if key.Name == serverlessPrefix+"test1" {
									return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
								}
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Labels = map[string]string{
						constants.LabelBizType: bizType,
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.VictoriaAccount.SupportServerless = true // Enable serverless support
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: false, PartResourceExists: true},
			},
		},
		"observeCompositeResource--ServerlessCluster-ServerlessIngressError": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							// Return error specifically for serverless ingress
							if _, ok := obj.(*k8snetworkv1.Ingress); ok {
								if key.Name == serverlessPrefix+"test1" {
									return fmt.Errorf("network error")
								}
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Labels = map[string]string{
						constants.LabelBizType: bizType,
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.VictoriaAccount.SupportServerless = true // Enable serverless support
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{},
				err:    fmt.Errorf("network error"),
			},
		},
		"ObserveStatusCreatingWaitEndpoint": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.CREATING
								promendpoint.Spec.Shared = true
								promendpoint.Spec.EndpointServiceID = "test1"
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.AvailabilityZone = "test1"
					res.Status.Phase = constants.CREATING
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pollInterval},
			},
		},
		"ObserveStatusCreatingWaitEndpoint-Shared": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.CREATING
								promendpoint.Spec.Shared = true
								promendpoint.Spec.EndpointServiceID = "test1"
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Status.Phase = constants.CREATING
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pollInterval},
			},
		},
		"ObserveStatusCreatingEndpointUpdate": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.CREATING
								promendpoint.Spec.Shared = true
								g.Expect(promendpoint.Spec.EndpointServiceID).Should(BeEmpty())
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.CREATING
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, PollInterval: pollInterval},
			},
		},
		"ObserveStatusCreatingIgnoresAgent": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if endpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								endpoint.Status.Phase = constants.ACTIVE
							}
							if pe, ok := obj.(*observabilitypingcapcomv1.PrivateEndpoint); ok {
								pe.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						EndpointNamePerfix: "test1",
					}
					res.Status.Phase = constants.CREATING
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true},
			},
		},
		"ObserveStatusCreatingReady": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.CREATING
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true},
			},
		},
		"ObserveStatusUpdatingIgnoresAgent": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if endpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								endpoint.Status.Phase = constants.ACTIVE
							}
							if pe, ok := obj.(*observabilitypingcapcomv1.PrivateEndpoint); ok {
								pe.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						EndpointNamePerfix: "test1",
					}
					res.Status.Phase = constants.UPDATING
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true},
			},
		},
		"ObserveStatusUpdatingTimeError": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = "test1"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{},
				err:    errtimeprase,
			},
		},
		"ObserveStatusUpdatingWaitOperator": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(-5 * time.Minute)),
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Format(time.RFC3339)
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pollInterval},
			},
		},
		"ObserveStatusUpdatingTimeout": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(-15 * time.Minute)),
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Add(-10 * time.Minute).Format(time.RFC3339)
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false},
			},
		},
		"ObserveStatusUpdatingTimeoutConditionFalse": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionFalse,
									LastTransitionTime: metav1.NewTime(NowTime.Add(-15 * time.Minute)),
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Add(-10 * time.Minute).Format(time.RFC3339)
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pollInterval},
			},
		},
		"ObserveStatusUpdatingWaitPrometheusReady": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionFalse,
									LastTransitionTime: metav1.NewTime(NowTime.Add(5 * time.Minute)),
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Format(time.RFC3339)
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pollInterval},
			},
		},
		"ObserveStatusUpdatingPrometheusReady": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(5 * time.Minute)),
								})
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Format(time.RFC3339)
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true},
			},
		},
		"ObserveStatusActiveEqualed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(5 * time.Minute)),
								})
								prome.Name = "test1"
								prome.Namespace = "test1"
								prome.Spec.Version = "v2.33.5"
								clusterLabels := map[string]string{
									constants.LabelTenantID:  "test-tenant1",
									constants.LabelProjectID: "test-project1",
									constants.LabelRegion:    "test-region1",
									constants.LabelVendor:    "aws",
									constants.LabelBizType:   "test",
								}
								prome.SetLabels(clusterLabels)
								prome.Spec.Replicas = pointer.Int32(1)
								prome.Spec.Retention = "15d"
								prome.Spec.EnableRemoteWriteReceiver = true
								prome.Spec.SecurityContext = getPodSecurityContext()

								prome.Spec.Resources = k8sapiv1.ResourceRequirements{
									Requests: k8sapiv1.ResourceList{
										"memory": resource.MustParse("1Gi"),
										"cpu":    resource.MustParse("2"),
									},
									Limits: k8sapiv1.ResourceList{
										"memory": resource.MustParse("1Gi"),
										"cpu":    resource.MustParse("2"),
									},
								}
								storage := resource.MustParse("20Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								initContainer := overrideContainer("init-config-reloader")
								container := overrideContainer("config-reloader")
								initContainer.Image = "test1.dkr.ecr.us-west-2.amazonaws.com/observability/prometheus-config-reloader:v0.57.0"
								container.Image = "test1.dkr.ecr.us-west-2.amazonaws.com/observability/prometheus-config-reloader:v0.57.0"
								prome.Spec.InitContainers = append(prome.Spec.InitContainers, initContainer)
								prome.Spec.Containers = append(prome.Spec.Containers, container)
								prome.Spec.NodeSelector = map[string]string{
									"node_type":                   "worker",
									"topology.kubernetes.io/zone": "test1",
								}
								prome.Spec.PodMetadata = &prometheusv1.EmbeddedObjectMetadata{
									Labels: clusterLabels,
								}
								prome.Spec.ExternalLabels = map[string]string{
									constants.ExternalLabelTenantID:  "test-tenant1",
									constants.ExternalLabelProjectID: "test-project1",
									constants.ExternalLabelRegion:    "aws-test-region1",
									constants.ExternalLabelBizType:   "test",
								}
								prome.Spec.EnableAdminAPI = true
								//if all Selector is nil, operator will not create prometheus config,then external lables will not work
								//ref https://github.com/prometheus-operator/prometheus-operator/blob/main/pkg/prometheus/operator.go#L1926
								prome.Spec.ServiceMonitorSelector = &metav1.LabelSelector{}
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Labels = map[string]string{
						constants.LabelTenantID:  "test-tenant1",
						constants.LabelProjectID: "test-project1",
						constants.LabelRegion:    "test-region1",
						constants.LabelVendor:    "aws",
						constants.LabelBizType:   "test",
					}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.ACTIVE
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Version = "v2.33.5"
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("1Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("2")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("20Gi")
					res.Spec.Prometheus.Replicas = 1
					res.Spec.Prometheus.Retention = "15d"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceUpToDate: true, ResourceExists: true},
			},
		},
		"ObserveStatusActiveEqualed-Shared": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.ACTIVE
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceUpToDate: false, ResourceExists: true},
			},
		},
		"ObserveStatusActiveEqualed-Agent": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							if _, ok := list.(*observabilitypingcapcomv1.PrivateEndpointList); ok {
								return nil
							}
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.ACTIVE
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = true
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "1.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 1,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 1,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 1,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 1,
								},
							},
						},
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceUpToDate: true, ResourceExists: true},
			},
		},
		"ObserveStatusActiveNotEqualed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(5 * time.Minute)),
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("1Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("2")
								storage := resource.MustParse("20Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "15d"

							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.ACTIVE
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true},
			},
		},

		"ObserveStatusPausedOfSharedCluster": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: true},
			},
		},
		"ObserveStatusPausedWaitPVCExpired": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = make(map[string]string)
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: fmt.Sprintf("%d", NowTime.Add(5*time.Minute).Unix()),
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false, PollInterval: pausedClusterPollInterval},
			},
		},
		"ObserveStatusPausedAndPVCExpiredGetPVCListFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),

						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = make(map[string]string)
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: "0",
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{},
				err:    errBoom,
			},
		},
		"ObserveStatusPausedAndPVCExpiredGetEmptyPVCList": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),

						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = make(map[string]string)
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: "0",
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceUpToDate: true, ResourceExists: true},
			},
		},
		"ObserveStatusPausedAndPVCExpiredGetNoneEmptyPVCList": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),

						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							pvclist, ok := list.(*k8sapiv1.PersistentVolumeClaimList)
							if !ok || pvclist == nil {
								return errBoom
							}

							pvclist.Items = []k8sapiv1.PersistentVolumeClaim{{}}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = make(map[string]string)
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: "0",
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true},
			},
		},
		"ObserveTO_BE_DELETED": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(5 * time.Minute)),
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("1Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("2")
								storage := resource.MustParse("20Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "15d"

							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.Phase = constants.ACTIVE
							}
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Status.Phase = constants.TO_BE_DELETED
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Spec.Prometheus.AvailabilityZone = "test1"
					res.Labels = map[string]string{}
					res.Annotations = map[string]string{
						constants.AnnotationNewDataSnapshot: "aaaa",
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: true},
			},
		},
	}
	for name, tc := range cases {
		for _, vendor := range []string{string(constants.AWS), string(constants.GCP)} {
			t.Run(name+"-"+vendor, func(t *testing.T) {
				g := NewGomegaWithT(t)
				ctl := gomock.NewController(t)
				defer ctl.Finish()
				azconfig := make(map[string]structs.AvailabilityZoneConfig)
				azconfig["test1"] = structs.AvailabilityZoneConfig{
					EndpointServiceID: "test1",
					DnsName:           "test1",
					IngressClassName:  "test1",
				}
				serviceinfo := structs.ServiceInfo{
					AZConfig: azconfig,
					AccountInfo: structs.AccountInfoConfig{
						Account: "test1",
					},
					Vendor: vendor,
					BizTypeConfigs: map[string]structs.BizTypeConfig{
						bizTypeServerless: {
							OmitTenancyReadIngress: structs.OmitTenancyReadIngress{
								AllOmit: true,
							},
						},
					},
					MonitorSpec: map[string]structs.MonitorSpec{
						"vmalert.default": {
							Cpu:            "10m",
							Memory:         "50Mi",
							InitialStorage: "10Gi",
						},
					},
				}
				configs.Init(serviceinfo)
				r := Cluster{
					cli:         tc.args.cli(g),
					serviceInfo: serviceinfo,
				}
				if tc.want.paniced {
					g.Expect(func() { r.Observe(context.Background(), tc.args.resource()) }).Should(PanicWith(errNotCluster))
				} else {
					got, err := r.Observe(context.Background(), tc.args.resource())
					if tc.want.err == nil {
						g.Expect(err).Should(BeNil())
					} else {
						g.Expect(err).Should(Equal(tc.want.err))
					}
					g.Expect(got).Should(Equal(tc.want.result))
				}

			})
		}
	}
}

func TestClusterControllerObserveLifecycleBoundaries(t *testing.T) {
	metrics.SetupMetrics()
	errBoom := errors.New("boom")

	t.Run("returns snapshot restart error", func(t *testing.T) {
		connector := &Cluster{
			cli: &fake.MockClient{
				MockGet: fake.NewMockGetFn(errBoom),
			},
		}
		cluster := &observabilitypingcapcomv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name:      "test",
				Namespace: "test",
				Annotations: map[string]string{
					constants.AnnotationNewDataSnapshot: "snapshot",
				},
			},
			Spec: observabilitypingcapcomv1.ClusterSpec{
				Prometheus: observabilitypingcapcomv1.Prometheus{Name: "test"},
			},
			Status: observabilitypingcapcomv1.ClusterStatus{Phase: constants.TO_BE_DELETED},
		}

		observation, err := connector.Observe(context.Background(), cluster)

		assert.ErrorIs(t, err, errBoom)
		assert.Equal(t, reconciler.ExternalObservation{}, observation)
	})

	t.Run("serverless cluster does not require dependent resources", func(t *testing.T) {
		connector := &Cluster{
			cli: fake.NewMockClient(),
			serviceInfo: structs.ServiceInfo{
				BizTypeConfigs: map[string]structs.BizTypeConfig{
					bizTypeServerless: {
						OmitTenancyReadIngress: structs.OmitTenancyReadIngress{AllOmit: true},
					},
				},
			},
		}
		cluster := &observabilitypingcapcomv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{constants.AnnotationShareMode: string(constants.SHARED)},
				Labels:      map[string]string{constants.LabelBizType: bizTypeServerless},
			},
			Spec: observabilitypingcapcomv1.ClusterSpec{
				VictoriaAccount: observabilitypingcapcomv1.VictoriaAccount{ProjectID: 1},
			},
		}

		observation, err := connector.Observe(context.Background(), cluster)

		assert.NoError(t, err)
		assert.Equal(t, reconciler.ExternalObservation{ResourceExists: true, ResourceUpToDate: false}, observation)
	})

	t.Run("unsupported availability zone is requeued", func(t *testing.T) {
		connector := &Cluster{
			cli:         fake.NewMockClient(),
			serviceInfo: structs.ServiceInfo{AZConfig: map[string]structs.AvailabilityZoneConfig{}},
			event:       record.NewFakeRecorder(1),
		}
		cluster := &observabilitypingcapcomv1.Cluster{
			Spec: observabilitypingcapcomv1.ClusterSpec{
				Prometheus: observabilitypingcapcomv1.Prometheus{AvailabilityZone: "unsupported"},
			},
			Status: observabilitypingcapcomv1.ClusterStatus{Phase: constants.ACTIVE},
		}

		observation, err := connector.Observe(context.Background(), cluster)

		assert.NoError(t, err)
		assert.Equal(t, reconciler.ExternalObservation{PollInterval: pollInterval}, observation)
	})
}

func TestClusterControllerCreate(t *testing.T) {
	configs.Init(structs.ServiceInfo{
		BizTypeConfigs: map[string]structs.BizTypeConfig{
			"test1": {
				MultiTenancyRule: false,
			},
			"test2": {
				MultiTenancyRule: true,
			},
		},
	})

	metrics.SetupMetrics()
	errBoom := errors.New("boom")
	noutfound := notfound

	cases := caseType{
		"CreateErrorResourceType": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{}
				},
				resource: func() client.Object {
					return &observabilitypingcapcomv1.S3{}
				},
			},
			want: want{
				err: errors.New(errNotCluster),
			},
		},
		"CreateUpdateStatusFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.CREATING))
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"CreateCreatePrometheusFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.CREATING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
						}),
						MockCreate: fake.MockCreateFn(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"CreateCreateSharedSuccess-Exclusive": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.CREATING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
						}),
						MockCreate: fake.MockCreateFn(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
							return nil
						}),
					}
				},
				serviceInfo: &structs.ServiceInfo{
					Vendor: string(constants.AWS),
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Spec.Prometheus.MetricsToObjectStorage = true
					res.Annotations = make(map[string]string)
					res.Annotations[constants.AnnotationServiceAccount] = "test1"
					res.Annotations[constants.AnnotationObjectStorageBucket] = "test1"
					res.Annotations[constants.AnnotationObjectStorageEndpoint] = "test1"
					res.Annotations[constants.AnnotationObjectStorageMetricsPrefix] = "test1"
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"CreateCreateSharedSuccess-Shared": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.CREATING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return noutfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
						}),
						MockCreate: fake.MockCreateFn(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.VictoriaAccount.AccountID = 111
					res.Spec.VictoriaAccount.Alerting = &prometheusv1.AlertingSpec{
						Alertmanagers: []prometheusv1.AlertmanagerEndpoints{
							{},
						},
					}
					res.Annotations = make(map[string]string)
					res.Annotations[constants.AnnotationServiceAccount] = "test1"
					res.Annotations[constants.AnnotationObjectStorageBucket] = "test1"
					res.Annotations[constants.AnnotationObjectStorageEndpoint] = "test1"
					res.Annotations[constants.AnnotationObjectStorageMetricsPrefix] = "test1"
					res.Annotations[constants.AnnotationShareMode] = string(constants.SHARED)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"CreateSharedWithAgentSuccess-Azure": {
			args: args{
				cli: func(g Gomega) client.Client {
					s := runtime.NewScheme()
					g.Expect(observabilitypingcapcomv1.AddToScheme(s)).Should(Succeed())
					g.Expect(prometheusv1.AddToScheme(s)).Should(Succeed())
					g.Expect(vmv1.AddToScheme(s)).Should(Succeed())
					g.Expect(k8sapiv1.AddToScheme(s)).Should(Succeed())
					cli := ctrlfake.NewClientBuilder().WithScheme(s).WithObjects(makeSeedCluster()).WithStatusSubresource(makeSeedCluster()).Build()
					return cli
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Labels = map[string]string{
						constants.LabelBizType:       bizType,
						constants.LabelApplicationID: constants.ApplicationIDK8sInfra,
					}
					res.Annotations = map[string]string{
						constants.LabelShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount = observabilitypingcapcomv1.VictoriaAccount{
						AvailabilityZone: "test1",
					}
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						ReleaseNameSpace:   "test1",
						EndpointNamePerfix: "test1",
						Azure: &observabilitypingcapcomv1.AgentAzure{
							SubscriptionID: "test1",
							ResourceGroup:  "test1",
							NetworkName:    "test1",
							Subnet:         "test1",
							IAMRoleRef: observabilitypingcapcomv1.Reference{
								Namespace: mockAzureIAM().Namespace,
								Name:      mockAzureIAM().Name,
							},
						},
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "7.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 1,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 1,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 1,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 1,
								},
							},
						},
					}
					return res
				},
				serviceInfo: &structs.ServiceInfo{
					Vendor: string(constants.Azure),
					AccountInfo: structs.AccountInfoConfig{
						SubscriptionID: "test0",
						ResourceGroup:  "test0",
					},
					AZConfig: map[string]structs.AvailabilityZoneConfig{
						"test1": {
							EndpointServiceID: "test0",
						},
					},
				},
			},
			want: want{
				err: nil,
			},
		},
		"CreateSharedWithAgent-Azure-PendingForPrivateEndpointReady": {
			args: args{
				cli: func(g Gomega) client.Client {
					s := runtime.NewScheme()
					g.Expect(observabilitypingcapcomv1.AddToScheme(s)).Should(Succeed())
					g.Expect(prometheusv1.AddToScheme(s)).Should(Succeed())
					g.Expect(vmv1.AddToScheme(s)).Should(Succeed())
					g.Expect(k8sapiv1.AddToScheme(s)).Should(Succeed())
					cli := ctrlfake.NewClientBuilder().WithScheme(s).WithObjects(makeSeedCluster()).WithStatusSubresource(makeSeedCluster()).Build()
					return cli
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Labels = map[string]string{
						constants.LabelBizType:       bizType,
						constants.LabelApplicationID: constants.ApplicationIDK8sInfra,
					}
					res.Annotations = map[string]string{
						constants.LabelShareMode: string(constants.SHARED),
					}
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						ReleaseNameSpace:   "test1",
						EndpointNamePerfix: "test1",
						Azure: &observabilitypingcapcomv1.AgentAzure{
							SubscriptionID: "test1",
							ResourceGroup:  "test1",
							NetworkName:    "test1",
							Subnet:         "test1",
						},
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "7.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 1,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 1,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 1,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 1,
								},
							},
						},
					}
					return res
				},
				serviceInfo: &structs.ServiceInfo{
					Vendor: string(constants.Azure),
					AZConfig: map[string]structs.AvailabilityZoneConfig{
						"test1": {
							EndpointServiceID: "test1",
						},
					},
				},
			},
			want: want{
				err: nil,
			},
		},
		"CreateSharedWithAgentSuccess-AWS": {
			args: args{
				cli: func(g Gomega) client.Client {
					s := runtime.NewScheme()
					g.Expect(observabilitypingcapcomv1.AddToScheme(s)).Should(Succeed())
					g.Expect(prometheusv1.AddToScheme(s)).Should(Succeed())
					g.Expect(vmv1.AddToScheme(s)).Should(Succeed())
					g.Expect(k8sapiv1.AddToScheme(s)).Should(Succeed())
					cli := ctrlfake.NewClientBuilder().WithScheme(s).WithObjects(makeSeedCluster()).WithStatusSubresource(makeSeedCluster()).Build()
					return cli
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Labels = map[string]string{
						constants.LabelBizType:       bizType,
						constants.LabelApplicationID: constants.ApplicationIDK8sInfra,
					}
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						ReleaseNameSpace:   "test1",
						EndpointNamePerfix: "test1",
						AWS: &observabilitypingcapcomv1.AgentAWS{
							VpcID:   "test1",
							Account: "test1",
						},
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "7.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 1,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 1,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 1,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 1,
								},
							},
						},
					}
					return res
				},
				serviceInfo: &structs.ServiceInfo{
					Vendor: string(constants.AWS),
					AZConfig: map[string]structs.AvailabilityZoneConfig{
						"test1": {
							EndpointServiceID: "test1",
						},
					},
				},
			},
			want: want{
				err: nil,
			},
		},
		"CreateSharedWithAgentSuccess-AWS-with-ProxyClusterID": {
			args: args{
				cli: func(g Gomega) client.Client {
					s := runtime.NewScheme()
					g.Expect(observabilitypingcapcomv1.AddToScheme(s)).Should(Succeed())
					g.Expect(prometheusv1.AddToScheme(s)).Should(Succeed())
					g.Expect(vmv1.AddToScheme(s)).Should(Succeed())
					g.Expect(k8sapiv1.AddToScheme(s)).Should(Succeed())
					cli := ctrlfake.NewClientBuilder().WithScheme(s).WithObjects(makeSeedProxyCluster()).WithStatusSubresource(makeSeedProxyCluster()).Build()
					return cli
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Labels = map[string]string{
						constants.LabelBizType:       bizType,
						constants.LabelApplicationID: constants.ApplicationIDK8sInfra,
					}
					res.Annotations = map[string]string{
						constants.AnnotationShareMode:      string(constants.SHARED),
						constants.AnnotationProxyClusterID: "test-proxy-cluster-id",
					}
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						ReleaseNameSpace:   "test1",
						EndpointNamePerfix: "test1",
						AWS: &observabilitypingcapcomv1.AgentAWS{
							VpcID:   "test1",
							Account: "test1",
						},
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "7.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 1,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 1,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 1,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 1,
								},
							},
						},
					}
					return res
				},
				serviceInfo: &structs.ServiceInfo{
					Vendor: string(constants.AWS),
					AZConfig: map[string]structs.AvailabilityZoneConfig{
						"test1": {
							EndpointServiceID: "test1",
						},
					},
				},
			},
			want: want{
				err: nil,
			},
		},
		"CreateSharedWithResourceRequirements": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.CREATING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return notfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
						}),
						MockCreate: fake.MockCreateFn(func(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AlertResources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.VictoriaAccount.AlertResources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.VictoriaAccount.AlertResources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.VictoriaAccount.AlertResources.Limits = make(k8sapiv1.ResourceList)
					res.Spec.VictoriaAccount.AlertResources.Limits[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.VictoriaAccount.AlertResources.Limits[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
	}
	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			g := NewGomegaWithT(t)
			ctl := gomock.NewController(t)
			defer ctl.Finish()
			r := Cluster{
				cli:   tc.args.cli(g),
				event: record.NewFakeRecorder(10),
			}
			if tc.args.serviceInfo != nil {
				r.serviceInfo = *tc.args.serviceInfo
			}
			res := tc.args.resource()
			if _, ok := r.cli.(*fake.MockClient); !ok {
				g.Expect(r.cli.Create(context.Background(), res)).To(Succeed())
			}
			if tc.want.paniced {
				g.Expect(func() { r.Create(context.Background(), res) }).Should(PanicWith(errNotCluster))
			} else {
				err := r.Create(context.Background(), res)
				if tc.want.err == nil {
					g.Expect(err).Should(BeNil())
				} else {
					g.Expect(err).Should(Equal(tc.want.err))
				}
			}

		})
	}
}

func TestClusterControllerUpdate(t *testing.T) {
	initConfig()
	metrics.SetupMetrics()
	errBoom := errors.New("boom")
	_, errtimeprase := time.Parse(time.RFC3339, "test1")
	NowTime := time.Now().UTC()
	cases := caseType{
		"UpdateErrorResourceType": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{}
				},
				resource: func() client.Object {
					return &observabilitypingcapcomv1.S3{}
				},
			},
			want: want{
				err: errors.New(errNotCluster),
			},
		},
		"GetMonitorCrdErrorWhenUpdate": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					return &observabilitypingcapcomv1.Cluster{}
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdateCreatingStatusFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("1Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("2")
								storage := resource.MustParse("20Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "15d"

							}
							if pvc, ok := obj.(*k8sapiv1.PersistentVolumeClaim); ok {
								pvc.Spec.VolumeName = "test1"
							}
							if pv, ok := obj.(*k8sapiv1.PersistentVolume); ok {
								pv.Labels = make(map[string]string)
								pv.Labels["topology.kubernetes.io/zone"] = "test1"
							}
							if promeingress, ok := obj.(*k8snetworkv1.Ingress); ok {
								promeingress.Spec.IngressClassName = utils.StringPtr("test1")
							}
							if promendpoint, ok := obj.(*observabilitypingcapcomv1.EndpointService); ok {
								promendpoint.Status.ServiceName = "test1"
								promendpoint.Status.Phase = constants.ACTIVE
							}

							return nil
						}),
						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							if podlist, ok := list.(*k8sapiv1.PodList); ok {
								podlist.Items = append(podlist.Items, k8sapiv1.Pod{
									Spec: k8sapiv1.PodSpec{
										Volumes: []k8sapiv1.Volume{
											{
												VolumeSource: k8sapiv1.VolumeSource{
													PersistentVolumeClaim: &k8sapiv1.PersistentVolumeClaimVolumeSource{
														ClaimName: "test1",
													},
												},
											},
										},
									},
								})
							}
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Status.Phase = constants.CREATING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdateClusterUpdatingParseTimeFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("1Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("2")
								storage := resource.MustParse("20Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "15d"

							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = "test1"
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: errtimeprase,
			},
		},
		"UpdateClusterUpdatingPrometheusReady": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(5 * time.Minute)),
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
								storage := resource.MustParse("40Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "30d"

							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Format(time.RFC3339)
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateClusterUpdatingTimeout": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:               prometheusv1.PrometheusAvailable,
									Status:             prometheusv1.PrometheusConditionTrue,
									LastTransitionTime: metav1.NewTime(NowTime.Add(-15 * time.Minute)),
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
								storage := resource.MustParse("40Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "30d"

							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					res.Status.Phase = constants.UPDATING
					res.Status.UpdateAt = NowTime.Add(-10 * time.Minute).Format(time.RFC3339)
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateClusterUpdatePrometheus": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								prome.Status.Conditions = append(prome.Status.Conditions, prometheusv1.PrometheusCondition{
									Type:   prometheusv1.PrometheusAvailable,
									Status: prometheusv1.PrometheusConditionTrue,
								})
								prome.Spec.Resources.Requests = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Limits = make(k8sapiv1.ResourceList)
								prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
								prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
								storage := resource.MustParse("40Gi")
								prome.Spec.Storage = getStorageSpec(&storage)
								prome.Spec.Retention = "30d"

							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok {
								g.Expect(prome.Spec.Resources.Requests[k8sapiv1.ResourceMemory]).Should(Equal(resource.MustParse("4Gi")))
								g.Expect(prome.Spec.Resources.Requests[k8sapiv1.ResourceCPU]).Should(Equal(resource.MustParse("8")))
								g.Expect(prome.Spec.Storage.VolumeClaimTemplate.Spec.Resources.Requests[k8sapiv1.ResourceStorage]).Should(Equal(resource.MustParse("80Gi")))
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("4Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("8")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("80Gi")
					res.Spec.Prometheus.Retention = "15d"
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateClusterActiveForSharedClusterWithAgent": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								vmalert.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							if vmrule, ok := obj.(*vmv1.VMRule); ok && vmrule != nil {
								vmrule.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "7.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 2,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 2,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 2,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 2,
								},
							},
						},
					}
					res.Status.Phase = constants.ACTIVE
					return res
				},
			},
			want: want{
				err: nil,
			},
		},

		"UpdateClusterActiveErrorForSharedClusterWithAgent": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								vmalert.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							if vmrule, ok := obj.(*vmv1.VMRule); ok && vmrule != nil {
								vmrule.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{}
					res.Status.Phase = constants.ACTIVE
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateClusterActiveErrorForSharedCluster": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								vmalert.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							if vmrule, ok := obj.(*vmv1.VMRule); ok && vmrule != nil {
								vmrule.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Status.Phase = constants.ACTIVE
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdatePausingStatusForSharedCluster": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								vmalert.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}
							if vmrule, ok := obj.(*vmv1.VMRule); ok && vmrule != nil {
								vmrule.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}

							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						EndpointNamePerfix: "test1",
					}
					res.Status.Phase = constants.PAUSING
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausingStatusForSharedClusterWithoutVMAlert": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmrule, ok := obj.(*vmv1.VMRule); ok && vmrule != nil {
								return notfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.MuteAlert = true
					res.Status.Phase = constants.PAUSING
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausingStatusMonitorHasNoReplicasFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								prome.Status.Replicas = 0
							}

							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdatePausingStatusMonitorHasNoReplicasSuccess": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								prome.Status.Replicas = 0
							}

							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausingStatusUpdateMonitorReplicasFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								replicas := int32(1)
								prome.Spec.Replicas = &replicas
								prome.Status.Replicas = replicas
							}

							return nil
						}),

						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Replicas = 0
					res.Status.Phase = constants.PAUSING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdatePausingStatusUpdateMonitorReplicasSuccess": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								replicas := int32(1)
								prome.Spec.Replicas = &replicas
								prome.Status.Replicas = replicas
							}

							return nil
						}),

						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Replicas = 0
					res.Status.Phase = constants.PAUSING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateResumingStatusForSharedCluster": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								vmalert.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}

							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.RESUMING
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.VictoriaAccount.AvailabilityZone = "test1"
					res.Spec.MuteAlert = false
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						EndpointNamePerfix: "test1",
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateResumingStatusForSharedClusterWithoutVMAlert": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmrule, ok := obj.(*vmv1.VMRule); ok {
								vmrule.Labels = map[string]string{
									constants.LabelClusterPaused: "true",
								}
							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.MuteAlert = true
					res.Status.Phase = constants.RESUMING
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateResumingStatusMonitorReadyFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								replicas := int32(1)
								prome.Spec.Replicas = &replicas
								prome.Status.Replicas = replicas
								prome.Status.AvailableReplicas = replicas
							}

							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return errBoom
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Replicas = 1
					res.Status.Phase = constants.RESUMING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdateResumingStatusMonitorReadySuccess": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								replicas := int32(1)
								prome.Spec.Replicas = &replicas
								prome.Status.Replicas = replicas
								prome.Status.AvailableReplicas = replicas
							}

							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							return nil
						}),
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Replicas = 1
					res.Status.Phase = constants.RESUMING
					res.Annotations = make(map[string]string)
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateResumingStatusUpdateMonitorReplicasFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								replicas := int32(0)
								prome.Spec.Replicas = &replicas
								prome.Status.Replicas = replicas
							}

							return nil
						}),

						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Replicas = 1
					res.Status.Phase = constants.RESUMING
					res.Annotations = make(map[string]string)
					res.Labels = map[string]string{
						constants.LabelBizType: "test1",
					}
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdateResumingStatusUpdateMonitorReplicasSuccess": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
								replicas := int32(0)
								prome.Spec.Replicas = &replicas
								prome.Status.Replicas = replicas
							}

							return nil
						}),

						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Replicas = 1
					res.Status.Phase = constants.RESUMING
					res.Annotations = make(map[string]string)
					res.Labels = map[string]string{
						constants.LabelBizType: "test1",
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausedStatusOfSharedCluster": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdateEmptyServerlessCluster": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {

							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = ""
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					res.Labels = map[string]string{
						constants.LabelBizType: bizTypeServerless,
					}
					res.Spec.VictoriaAccount.ProjectID = 100
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausedStatusWithPVCNotExpired": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}

							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = map[string]string{}
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: fmt.Sprintf("%d", NowTime.Unix()),
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausedStatusWithPVCExpiredGetPVCListFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}

							return nil
						}),

						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = map[string]string{}
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: "0",
					}
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdatePausedStatusWithPVCExpiredDeletePVCListSuccess": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}

							return nil
						}),

						MockDelete: fake.MockDeleteFn(func(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
							return nil
						}),

						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							pvclist, ok := list.(*k8sapiv1.PersistentVolumeClaimList)
							if !ok || pvclist == nil {
								return errBoom
							}

							pvclist.Items = []k8sapiv1.PersistentVolumeClaim{{}}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = map[string]string{}
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: "0",
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
		"UpdatePausedStatusWithPVCExpiredDeletePVCListFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if prome, ok := obj.(*prometheusv1.Prometheus); ok && prome != nil {
								prome.Labels = map[string]string{
									constants.LabelBizType: "test1",
								}
							}

							return nil
						}),

						MockDelete: fake.MockDeleteFn(func(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
							return errBoom
						}),

						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							pvclist, ok := list.(*k8sapiv1.PersistentVolumeClaimList)
							if !ok || pvclist == nil {
								return errBoom
							}

							item := k8sapiv1.PersistentVolumeClaim{}
							item.Namespace = "test1"
							pvclist.Items = []k8sapiv1.PersistentVolumeClaim{item}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Status.Phase = constants.PAUSED
					res.Annotations = map[string]string{}
					res.Labels = map[string]string{
						constants.LabelTimestampToPause: "0",
					}
					return res
				},
			},
			want: want{
				err: errBoom,
			},
		},
		"UpdateClusterWithNonSharedVMAlterWhenMultiTenancyEnabled": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockUpdate: fake.MockUpdateFn(func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								if vmalert.Spec.ReplicaCount == nil || *vmalert.Spec.ReplicaCount != 0 {
									return errBoom
								}
							}
							return nil
						}),
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.CREATING))
							return errBoom
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if vmrule, ok := obj.(*vmv1.VMRule); ok && vmrule != nil {
								vmrule.Labels = map[string]string{
									constants.LabelBizType:          "test1",
									constants.LabelMultiTenancyRule: "",
								}
							}
							if vmalert, ok := obj.(*vmv1.VMAlert); ok && vmalert != nil {
								replica := int32(1)
								vmalert.Spec.ReplicaCount = &replica
							}
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test2"
					res.Namespace = "test2"
					res.Status.Phase = constants.PAUSING
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					return res
				},
			},
			want: want{
				err: nil,
			},
		},
	}
	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			g := NewGomegaWithT(t)
			ctl := gomock.NewController(t)
			defer ctl.Finish()
			azconfig := make(map[string]structs.AvailabilityZoneConfig)
			azconfig["test1"] = structs.AvailabilityZoneConfig{
				EndpointServiceID: "test1",
				IngressClassName:  "test1",
			}
			serviceinfo := structs.ServiceInfo{
				AZConfig: azconfig,
				BizTypeConfigs: map[string]structs.BizTypeConfig{
					bizTypeServerless: {
						OmitTenancyReadIngress: structs.OmitTenancyReadIngress{
							AllOmit: true,
						},
					},
				},
			}
			r := Cluster{
				serviceInfo: serviceinfo,
				cli:         tc.args.cli(g),
				event:       record.NewFakeRecorder(10),
			}
			if tc.want.paniced {
				g.Expect(func() { r.Update(context.Background(), tc.args.resource()) }).Should(PanicWith(errNotCluster))
			} else {
				err := r.Update(context.Background(), tc.args.resource())
				if tc.want.err == nil {
					g.Expect(err).Should(BeNil())
				} else {
					g.Expect(err).Should(Equal(tc.want.err))
				}
			}

		})
	}
}

func TestClusterControllerUpdateCreatingLifecycle(t *testing.T) {
	metrics.SetupMetrics()
	errBoom := errors.New("boom")
	serviceInfo := structs.ServiceInfo{
		AZConfig: map[string]structs.AvailabilityZoneConfig{
			"test": {DnsName: "test.example.com"},
		},
	}
	newServerlessCluster := func() *observabilitypingcapcomv1.Cluster {
		return &observabilitypingcapcomv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Name:        "test",
				Annotations: map[string]string{constants.AnnotationShareMode: string(constants.SHARED)},
			},
			Spec: observabilitypingcapcomv1.ClusterSpec{
				VictoriaAccount: observabilitypingcapcomv1.VictoriaAccount{
					AvailabilityZone: "test",
					ProjectID:        1,
				},
			},
			Status: observabilitypingcapcomv1.ClusterStatus{Phase: constants.CREATING},
		}
	}

	t.Run("serverless cluster becomes active", func(t *testing.T) {
		cluster := newServerlessCluster()
		connector := &Cluster{cli: fake.NewMockClient(), serviceInfo: serviceInfo}

		err := connector.updateClusterCreating(context.Background(), ctrl.Log, cluster)

		assert.NoError(t, err)
		assert.Equal(t, constants.ACTIVE, cluster.Status.Phase)
		assert.Equal(t, "test.example.com", cluster.Status.DnsName)
	})

	t.Run("returns serverless status update error", func(t *testing.T) {
		cluster := newServerlessCluster()
		mockClient := fake.NewMockClient()
		mockClient.MockStatusUpdate = fake.NewMockStatusUpdateFn(errBoom)
		connector := &Cluster{cli: mockClient, serviceInfo: serviceInfo}

		err := connector.updateClusterCreating(context.Background(), ctrl.Log, cluster)

		assert.ErrorIs(t, err, errBoom)
	})

	t.Run("returns endpoint service get error", func(t *testing.T) {
		mockClient := fake.NewMockClient()
		mockClient.MockGet = fake.NewMockGetFn(errBoom)
		connector := &Cluster{cli: mockClient}
		cluster := &observabilitypingcapcomv1.Cluster{
			Spec:   observabilitypingcapcomv1.ClusterSpec{Endpoint: observabilitypingcapcomv1.Endpoint{Name: "test"}},
			Status: observabilitypingcapcomv1.ClusterStatus{Phase: constants.CREATING},
		}

		err := connector.updateClusterCreating(context.Background(), ctrl.Log, cluster)

		assert.ErrorIs(t, err, errBoom)
	})

	t.Run("waits for inactive endpoint service", func(t *testing.T) {
		connector := &Cluster{cli: fake.NewMockClient()}
		cluster := &observabilitypingcapcomv1.Cluster{
			Spec:   observabilitypingcapcomv1.ClusterSpec{Endpoint: observabilitypingcapcomv1.Endpoint{Name: "test"}},
			Status: observabilitypingcapcomv1.ClusterStatus{Phase: constants.CREATING},
		}

		err := connector.updateClusterCreating(context.Background(), ctrl.Log, cluster)

		assert.NoError(t, err)
		assert.Equal(t, constants.CREATING, cluster.Status.Phase)
	})
}

func TestClusterControllerLifecycleErrorPaths(t *testing.T) {
	initConfig()
	metrics.SetupMetrics()
	errBoom := errors.New("boom")

	t.Run("returns updating status error", func(t *testing.T) {
		mockClient := fake.NewMockClient()
		mockClient.MockStatusUpdate = fake.NewMockStatusUpdateFn(errBoom)
		connector := &Cluster{cli: mockClient}
		cluster := &observabilitypingcapcomv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{constants.AnnotationShareMode: string(constants.SHARED)}},
			Status:     observabilitypingcapcomv1.ClusterStatus{Phase: constants.UPDATING},
		}

		err := connector.updateClusterUpdating(context.Background(), ctrl.Log, cluster, nil)

		assert.ErrorIs(t, err, errBoom)
		assert.Equal(t, constants.ACTIVE, cluster.Status.Phase)
	})

	t.Run("returns empty status update error", func(t *testing.T) {
		mockClient := fake.NewMockClient()
		mockClient.MockStatusUpdate = fake.NewMockStatusUpdateFn(errBoom)
		connector := &Cluster{
			cli: mockClient,
			serviceInfo: structs.ServiceInfo{
				AZConfig: map[string]structs.AvailabilityZoneConfig{"test": {DnsName: "test.example.com"}},
				BizTypeConfigs: map[string]structs.BizTypeConfig{
					bizTypeServerless: {
						OmitTenancyReadIngress: structs.OmitTenancyReadIngress{AllOmit: true},
					},
				},
			},
		}
		cluster := &observabilitypingcapcomv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{constants.AnnotationShareMode: string(constants.SHARED)},
				Labels:      map[string]string{constants.LabelBizType: bizTypeServerless},
			},
			Spec: observabilitypingcapcomv1.ClusterSpec{
				VictoriaAccount: observabilitypingcapcomv1.VictoriaAccount{AvailabilityZone: "test", ProjectID: 1},
			},
		}

		err := connector.updateClusterEmpty(context.Background(), ctrl.Log, cluster)

		assert.ErrorIs(t, err, errBoom)
	})

	t.Run("rejects invalid monitor while pausing", func(t *testing.T) {
		connector := &Cluster{cli: fake.NewMockClient()}
		cluster := &observabilitypingcapcomv1.Cluster{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}}

		err := connector.updateClusterPausing(context.Background(), ctrl.Log, cluster, &observabilitypingcapcomv1.S3{})

		assert.Error(t, err)
	})

	t.Run("rejects invalid monitor while resuming", func(t *testing.T) {
		connector := &Cluster{cli: fake.NewMockClient()}
		cluster := &observabilitypingcapcomv1.Cluster{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{}}}

		err := connector.updateClusterResuming(context.Background(), ctrl.Log, cluster, &observabilitypingcapcomv1.S3{})

		assert.Error(t, err)
	})

	t.Run("returns snapshot restart error while resuming", func(t *testing.T) {
		mockClient := fake.NewMockClient()
		mockClient.MockGet = fake.NewMockGetFn(errBoom)
		connector := &Cluster{cli: mockClient}
		cluster := &observabilitypingcapcomv1.Cluster{
			ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{constants.AnnotationNewDataSnapshot: "snapshot"}},
			Spec:       observabilitypingcapcomv1.ClusterSpec{Prometheus: observabilitypingcapcomv1.Prometheus{Name: "test"}},
			Status:     observabilitypingcapcomv1.ClusterStatus{Phase: constants.RESUMING},
		}
		prometheus := &prometheusv1.Prometheus{
			ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{constants.LabelBizType: "test1"}},
			Status:     prometheusv1.PrometheusStatus{Replicas: 1, AvailableReplicas: 1},
		}

		err := connector.updateClusterResuming(context.Background(), ctrl.Log, cluster, prometheus)

		assert.ErrorIs(t, err, errBoom)
	})

	t.Run("get and create returns get error", func(t *testing.T) {
		connector := &Cluster{cli: &fake.MockClient{MockGet: fake.NewMockGetFn(errBoom)}}

		err := connector.getAndCreate(context.Background(), "test", "test", &observabilitypingcapcomv1.EndpointService{}, &observabilitypingcapcomv1.Cluster{}, makeEndpointService)

		assert.ErrorIs(t, err, errBoom)
	})

	t.Run("get and create returns resource maker error", func(t *testing.T) {
		mockClient := fake.NewMockClient()
		mockClient.MockGet = fake.NewMockGetFn(notfound(schema.GroupVersionKind{}, "test"))
		connector := &Cluster{cli: mockClient}

		err := connector.getAndCreate(
			context.Background(),
			"test",
			"test",
			&observabilitypingcapcomv1.EndpointService{},
			&observabilitypingcapcomv1.Cluster{},
			func(*Cluster, *observabilitypingcapcomv1.Cluster) (client.Object, error) { return nil, errBoom },
		)

		assert.ErrorIs(t, err, errBoom)
	})
}

func TestClusterControllerDelete(t *testing.T) {
	metrics.SetupMetrics()
	errBoom := errors.New("boom")
	cases := caseType{
		"DeleteErrorResourceType": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{}
				},
				resource: func() client.Object {
					return &observabilitypingcapcomv1.S3{}
				},
			},
			want: want{
				result: reconciler.ExternalDeletion{},
				err:    errors.New(errNotCluster),
			},
		},
		"DeleteUpdateSattusFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.DELETING))
							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalDeletion{},
				err:    errBoom,
			},
		},
		"DeleteIngressFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.DELETING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return errBoom
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalDeletion{},
				err:    errBoom,
			},
		},
		"DeleteMonitorFailed": {
			args: args{
				cli: func(g Gomega) client.Client {
					mockClient := fake.NewMockClient()
					mockClient.MockGet = fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
						switch obj.(type) {
						case *observabilitypingcapcomv1.EndpointService:
							return notfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
						case *k8sapiv1.Service:
							return errBoom
						default:
							return nil
						}
					})
					return mockClient
				},
				resource: func() client.Object {
					return &observabilitypingcapcomv1.Cluster{
						ObjectMeta: metav1.ObjectMeta{Name: "test", Namespace: "test"},
						Spec: observabilitypingcapcomv1.ClusterSpec{
							Endpoint:   observabilitypingcapcomv1.Endpoint{Name: "test"},
							Prometheus: observabilitypingcapcomv1.Prometheus{Name: "test"},
						},
						Status: observabilitypingcapcomv1.ClusterStatus{Phase: constants.DELETING},
					}
				},
			},
			want: want{
				result: reconciler.ExternalDeletion{},
				err:    errBoom,
			},
		},
		"DeleteSuccessExclusive": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.DELETING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {

							return nil
						}),
						MockDelete: fake.MockDeleteFn(func(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
							return nil
						}),
						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.Prometheus.Resources.Requests = make(k8sapiv1.ResourceList)
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceMemory] = resource.MustParse("2Gi")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceCPU] = resource.MustParse("4")
					res.Spec.Prometheus.Resources.Requests[k8sapiv1.ResourceStorage] = resource.MustParse("40Gi")
					res.Spec.Prometheus.Retention = "30d"
					return res
				},
			},
			want: want{
				result: reconciler.ExternalDeletion{ResourceDeleted: true},
			},
		},
		"DeleteSuccessShared": {
			args: args{
				cli: func(g Gomega) client.Client {
					return &fake.MockClient{
						MockStatusUpdate: fake.MockStatusUpdateFn(func(ctx context.Context, obj client.Object, opts ...client.SubResourceUpdateOption) error {
							res := obj.(*observabilitypingcapcomv1.Cluster)
							g.Expect(res.Status.Phase).Should(Equal(constants.DELETING))
							return nil
						}),
						MockGet: fake.MockGetFn(func(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
							if _, ok := obj.(*observabilitypingcapcomv1.PrivateEndpoint); ok {
								return notfound(obj.GetObjectKind().GroupVersionKind(), key.Name)
							}
							return nil
						}),
						MockDelete: fake.MockDeleteFn(func(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
							return nil
						}),
						MockList: fake.MockListFn(func(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
							return nil
						}),
					}
				},
				resource: func() client.Object {
					res := &observabilitypingcapcomv1.Cluster{}
					res.Name = "test1"
					res.Namespace = "test1"
					res.Spec.Prometheus.Name = "test1"
					res.Spec.Endpoint.Name = "test1"
					res.Spec.Endpoint.Shared = true
					res.Spec.VictoriaAccount.VMClusterName = "aaa"
					res.Spec.Agent = &observabilitypingcapcomv1.Agent{
						Observed: &observabilitypingcapcomv1.Observed{
							TiDBVersion: "1.0",
							TiDBClusterTopology: &observabilitypingcapcomv1.ClusterTopologyType{
								TiDB: observabilitypingcapcomv1.TiDB{
									Replicas: 1,
								},
								TiKV: observabilitypingcapcomv1.TiKV{
									Replicas: 1,
								},
								PD: observabilitypingcapcomv1.PD{
									Replicas: 1,
								},
								TiFlash: &observabilitypingcapcomv1.TiFlash{
									Replicas: 1,
								},
							},
						},
					}
					res.Annotations = map[string]string{
						constants.AnnotationShareMode: string(constants.SHARED),
					}
					return res
				},
			},
			want: want{
				result: reconciler.ExternalDeletion{ResourceDeleted: true},
			},
		},
	}
	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			g := NewGomegaWithT(t)
			ctl := gomock.NewController(t)
			defer ctl.Finish()
			r := Cluster{
				cli:   tc.args.cli(g),
				event: record.NewFakeRecorder(10),
			}
			if tc.want.paniced {
				g.Expect(func() { r.Delete(context.Background(), tc.args.resource()) }).Should(PanicWith(errNotCluster))
			} else {
				got, err := r.Delete(context.Background(), tc.args.resource())
				if tc.want.err == nil {
					g.Expect(err).Should(BeNil())
				} else {
					g.Expect(err).Should(Equal(tc.want.err))
				}
				g.Expect(got).Should(Equal(tc.want.result))
			}

		})
	}
}

func TestMakePrometheusIngress(t *testing.T) {
	g := NewGomegaWithT(t)
	ingressname := "testingress"
	ingressnamespace := "testnamespace"
	ingressClassName := "class1"
	endpointname := "endp1"
	var promeingress k8snetworkv1.Ingress
	promeingress.SetName(ingressname)
	promeingress.SetNamespace(ingressnamespace)
	promeingress.Spec.IngressClassName = utils.StringPtr(ingressClassName)
	promeingress.Annotations = make(map[string]string)
	promeingress.Annotations["nginx.ingress.kubernetes.io/rewrite-target"] = "/$2"
	promeingress.Annotations["nginx.ingress.kubernetes.io/use-regex"] = "true"
	var rule k8snetworkv1.IngressRule
	var httprule k8snetworkv1.HTTPIngressRuleValue
	pathtype := k8snetworkv1.PathTypeImplementationSpecific
	httprule.Paths = append(httprule.Paths, k8snetworkv1.HTTPIngressPath{
		Path:     "/endp1" + "(/|$)(.*)",
		PathType: &pathtype,
		Backend: k8snetworkv1.IngressBackend{
			Service: &k8snetworkv1.IngressServiceBackend{
				Name: svcPrefix + endpointname,
				Port: k8snetworkv1.ServiceBackendPort{
					Number: prometheusPort,
				},
			},
		},
	})
	rule.HTTP = &httprule
	promeingress.Spec.Rules = append(promeingress.Spec.Rules, rule)
	g.Expect(makePrometheusIngress(ingressname, ingressnamespace, ingressClassName, "/endp1", endpointname, "")).Should(Equal(promeingress))
}

func TestMakeVMIngress(t *testing.T) {
	g := NewGomegaWithT(t)
	ingressname := "testingress"
	ingressnamespace := "testnamespace"
	ingressClassName := "class1"
	var vmingress k8snetworkv1.Ingress
	vmingress.SetName(ingressname)
	vmingress.SetNamespace(ingressnamespace)
	vmingress.Spec.IngressClassName = utils.StringPtr(ingressClassName)
	vmingress.Annotations = make(map[string]string)
	vmingress.Annotations["nginx.ingress.kubernetes.io/use-regex"] = "true"
	vmingress.Annotations["nginx.ingress.kubernetes.io/rewrite-target"] = "/insert/111:1/prometheus/$2"
	var rule k8snetworkv1.IngressRule
	var httprule k8snetworkv1.HTTPIngressRuleValue
	pathtype := k8snetworkv1.PathTypeImplementationSpecific
	httprule.Paths = append(httprule.Paths, k8snetworkv1.HTTPIngressPath{
		Path:     "/clusterid" + "(/|$)(.*)",
		PathType: &pathtype,
		Backend: k8snetworkv1.IngressBackend{
			Service: &k8snetworkv1.IngressServiceBackend{
				Name: "vmselect-11111",
				Port: k8snetworkv1.ServiceBackendPort{
					Number: vmInsertPort,
				},
			},
		},
	})
	rule.HTTP = &httprule
	vmingress.Spec.Rules = append(vmingress.Spec.Rules, rule)
	g.Expect(makeVMIngress(ingressname, ingressnamespace, ingressClassName, vmInsertAPI, 111, 1, "/clusterid",
		"vmselect-11111", vmInsertPort, "", false)).Should(Equal(vmingress))
}

func TestMakeThanosObjectConfigSecret(t *testing.T) {
	g := NewGomegaWithT(t)

	cases := map[string]struct {
		serviceInfo structs.ServiceInfo
		cluster     *observabilitypingcapcomv1.Cluster
	}{
		"aws": {
			serviceInfo: structs.ServiceInfo{
				Vendor: string(constants.AWS),
			},
			cluster: &observabilitypingcapcomv1.Cluster{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test1",
					Namespace: "test1",
					Annotations: map[string]string{
						constants.AnnotationObjectStorageBucket:        "test1",
						constants.AnnotationObjectStorageEndpoint:      "test1",
						constants.AnnotationObjectStorageMetricsPrefix: "test1",
					},
				},
			},
		},
		"gcp": {
			serviceInfo: structs.ServiceInfo{
				Vendor: string(constants.GCP),
			},
			cluster: &observabilitypingcapcomv1.Cluster{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test1",
					Namespace: "test1",
					Annotations: map[string]string{
						constants.AnnotationObjectStorageBucket:        "test1",
						constants.AnnotationObjectStorageMetricsPrefix: "test1",
					},
				},
			},
		},
		"azure": {
			serviceInfo: structs.ServiceInfo{
				Vendor: string(constants.Azure),
			},
			cluster: &observabilitypingcapcomv1.Cluster{
				ObjectMeta: metav1.ObjectMeta{
					Name:      "test1",
					Namespace: "test1",
					Annotations: map[string]string{
						constants.AnnotationObjectStorageBucket:        "test1",
						constants.AnnotationBlobStorageAccount:         "test1",
						constants.AnnotationObjectStorageMetricsPrefix: "test1",
					},
				},
			},
		},
	}

	for name, tc := range cases {
		t.Run(name, func(t *testing.T) {
			cls := &Cluster{
				serviceInfo: tc.serviceInfo,
			}
			_, err := makeThanosObjectConfigSecret(cls, tc.cluster)
			g.Expect(err).To(BeNil())
		})
	}
}

func makeSeedCluster() *observabilitypingcapcomv1.Cluster {
	return &observabilitypingcapcomv1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:      bizType,
			Namespace: bizType,
			Labels: map[string]string{
				constants.LabelBizType:       bizType,
				constants.LabelApplicationID: constants.ApplicationIDSeed,
				constants.LabelProjectID:     constants.ApplicationIDSeed,
			},
		},
	}
}

func makeSeedProxyCluster() *observabilitypingcapcomv1.Cluster {
	return &observabilitypingcapcomv1.Cluster{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test-proxy-cluster-id",
			Namespace: bizType,
			Labels: map[string]string{
				constants.LabelBizType:       bizType,
				constants.LabelApplicationID: "cdcd",
				constants.LabelProjectID:     constants.ApplicationIDSeed,
				constants.LabelClusterID:     "test-proxy-cluster-id",
			},
		},
	}
}

func mockAzureIAM() *observabilitypingcapcomv1.IAM {
	return &observabilitypingcapcomv1.IAM{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "test1",
			Namespace: "test1",
			Labels: map[string]string{
				constants.LabelBizType:       bizType,
				constants.LabelApplicationID: constants.ApplicationIDSeed,
				constants.LabelProjectID:     constants.ApplicationIDSeed,
			},
		},
		Status: observabilitypingcapcomv1.IAMStatus{
			Phase:    constants.ACTIVE,
			ClientId: "test1",
		},
	}
}