//go:build intergration

package ft

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	containerpb "cloud.google.com/go/container/apiv1/containerpb"
	"github.com/aws/aws-sdk-go-v2/service/ec2"
	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
	"github.com/aws/aws-sdk-go-v2/service/eks"
	ekstypes "github.com/aws/aws-sdk-go-v2/service/eks/types"
	"github.com/aws/aws-sdk-go/aws"
	ec2v1beta1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1beta1"
	computev1alpha1 "github.com/crossplane-contrib/provider-gcp/apis/compute/v1alpha1"
	"google.golang.org/api/googleapi"
	apierrors "k8s.io/apimachinery/pkg/api/errors"

	"cloud.google.com/go/compute/apiv1/computepb"
	"github.com/crossplane-contrib/provider-aws/apis/eks/manualv1alpha1"
	eksv1alpha1 "github.com/crossplane-contrib/provider-aws/apis/eks/v1alpha1"
	eksv1beta1 "github.com/crossplane-contrib/provider-aws/apis/eks/v1beta1"
	gkev1beta1 "github.com/crossplane-contrib/provider-gcp/apis/container/v1beta1"
	gkev1beta2 "github.com/crossplane-contrib/provider-gcp/apis/container/v1beta2"
	. "github.com/onsi/ginkgo/v2"
	. "github.com/onsi/gomega"

	regionalpb "github.com/tidbcloud/infra-provider/api/tidbcloud/regional-server/v1"
	"github.com/tidbcloud/infra-provider/pkg/consts"
	"github.com/tidbcloud/infra-provider/pkg/namer"
	rsv1alpha1 "github.com/tidbcloud/infra-provider/regional-server/api/v1alpha1"
	"github.com/tidbcloud/infra-provider/regional-server/pkg/common/constants"
	"github.com/tidbcloud/infra-provider/regional-server/pkg/common/utils"
	"github.com/tidbcloud/infra-provider/regional-server/pkg/controllers/appbox/task/gke"
	"golang.org/x/sync/errgroup"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

const (
	Nonexistent = "nonexistent"
	interval    = 30 * time.Second
	GNotFound   = 404
)

var errWait = fmt.Errorf("wait for condition met")

var _ = Describe("Create Clusters API Test", Label("CreateCluster"), func() {
	// runs in order, avoid expensive setup
	Context("Base Cases In Order", Ordered, func() {
		var err error
		var vpcName, vpcId string
		var c *regionalpb.CreateClusterRequest
		BeforeAll(func() {
			req := CreateVPCReq(f.TenantFoo, f.ProjectFoo)
			vpcName = utils.MD5(req.RequestId)
			_, err = vpcClient.CreateVpc(ctx, req)
			Expect(err).To(BeNil())
			By("Wait For VPC Ready Within 5 mins")
			vpcCR := &rsv1alpha1.VPC{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName,
					Namespace: f.Config.Namespace,
				},
			}
			waitVPCReady(ctx, vpcCR, 5*time.Minute, interval)
			vpcId = vpcCR.Status.VPC.VPCID
		})
		It("Should Create Cluster Successfully", func() {
			By("Create Cluster With Same VPCID and ClusterID Concurrently")
			c = CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			errG, _ := errgroup.WithContext(ctx)
			for i := 0; i < 3; i++ {
				errG.Go(func() error {
					_, err := clusterClient.CreateCluster(ctx, c)
					if err != nil && !strings.Contains(err.Error(), "HTTPSTATUSCODE:409") {
						return fmt.Errorf("unexpected Http Code: %w", err)
					}
					return nil
				})
			}
			Expect(errG.Wait()).To(BeNil(), "request %v", c)
			By("Check RegionalServer CR Status")
			By("Wait For APPBox Ready Within 30 mins")
			appboxCR := &rsv1alpha1.APPBox{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPBoxReady(ctx, appboxCR, 30*time.Minute, interval)
			By("Wait For Internet Ready Within 5 Mins When appbox is ready")
			internetCR := &rsv1alpha1.Internet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			waitInternetReady(ctx, internetCR, 5*time.Minute, interval)
			By("Wait For APPRoute Ready Within 5 Mins")
			appRouteCR := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      c.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRouteCR, 10*time.Minute, interval)
			By("List VPC By Cluster")
			in := new(regionalpb.ListVpcsRequest)
			in.TenantId = c.Cluster.TenantId
			in.ProjectId = c.Cluster.ProjectId
			in.ClusterId = c.Cluster.Id
			out, err := vpcClient.ListVpcs(ctx, in)
			Expect(err).To(BeNil())
			Expect(out.Vpcs).To(HaveLen(1))
			Expect(out.Vpcs[0].TenantIds).To(ContainElement(in.TenantId))
			Expect(out.Vpcs[0].ProjectIds).To(ContainElement(in.ProjectId))
			switch f.Config.Provider {
			case "aws":
				By("Call QueryVpcClusterIds API")
				awsInfo := out.Vpcs[0].GetAws()
				Expect(awsInfo).NotTo(BeNil())
				in := new(regionalpb.QueryVpcClusterIdsRequest)
				in.TenantId = c.Cluster.TenantId
				in.ProjectId = c.Cluster.ProjectId
				in.Info = &regionalpb.QueryVpcClusterIdsRequest_Aws{
					Aws: &regionalpb.Vpc_AwsVpcInfo{
						VpcId:     awsInfo.VpcId,
						AccountId: awsInfo.AccountId,
					},
				}
				out, err := vpcClient.QueryVpcClusterIds(ctx, in)
				Expect(err).To(BeNil())
				Expect(out.ClusterIds).To(HaveLen(1))
				Expect(out.ClusterIds[0]).To(Equal(c.Cluster.Id))
				By("Check Crossplane CR: EKS Status")
				eksCluster := new(eksv1beta1.Cluster)
				eksCluster.Name = namer.UniqueIdentifier(appboxCR.ObjectMeta)
				Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(eksCluster), eksCluster)).To(Succeed())
				ready, synced := utils.GetConditions(eksCluster.Status.Conditions)
				Expect(ready && synced).To(BeTrue())
				By("Check Crossplane CR: NodeGroup Status")
				ng := new(manualv1alpha1.NodeGroup)
				ng.Name = namer.UniqueIdentifier(appboxCR.ObjectMeta, namer.WithPrefix(consts.TiDBCloudPrefix))
				Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(ng), ng)).To(Succeed())
				ready, synced = utils.GetConditions(ng.Status.Conditions)
				Expect(ready && synced).To(BeTrue())
				By("Check Crossplane CR: EKS Addons Status")
				addons := []string{"coredns", "vpc-cni", "kube-proxy", "aws-ebs-csi-driver"}
				for _, name := range addons {
					addon := new(eksv1alpha1.Addon)
					addon.Name = namer.UniqueIdentifier(appboxCR.ObjectMeta, namer.WithPrefix(name))
					Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(addon), addon)).To(Succeed())
					ready, synced = utils.GetConditions(addon.Status.Conditions)
					Expect(ready && synced).To(BeTrue())
				}
				By("Check Crossplane CR: IGW Status")
				igw := new(ec2v1beta1.InternetGateway)
				igw.Name = namer.UniqueIdentifier(internetCR.ObjectMeta)
				Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(igw), igw)).To(Succeed())
				ready, synced = utils.GetConditions(igw.Status.Conditions)
				Expect(ready && synced).To(BeTrue())

				By("Check Cloud Provider Resource: VPC Status")
				vpcOut, err := ec2Client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{
					VpcIds: []string{vpcId},
				})
				Expect(err).To(BeNil())
				Expect(vpcOut.Vpcs).To(HaveLen(1))
				Expect(vpcOut.Vpcs[0].State).To(Equal(ec2types.VpcStateAvailable))
				By("Check Cloud Provider Resource: VPC Subnet Status")
				snOut, err := ec2Client.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{
					Filters: []ec2types.Filter{
						{
							Name:   aws.String("vpc-id"),
							Values: []string{vpcId},
						},
					},
				})
				Expect(err).To(BeNil())
				// at least private, public, node subnet
				Expect(len(snOut.Subnets) > 2).Should(BeTrue())
				for _, sn := range snOut.Subnets {
					Expect(sn.State).To(Equal(ec2types.SubnetStateAvailable))
				}
				By("Check Cloud Provider Resource: IGW Status")
				igwOut, err := ec2Client.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{
					Filters: []ec2types.Filter{
						{
							Name:   aws.String("attachment.vpc-id"),
							Values: []string{vpcId},
						},
					},
				})
				Expect(err).To(BeNil())
				Expect(igwOut.InternetGateways).To(HaveLen(1))
				By("Check Cloud Provider Resource: NAT Status")
				natOut, err := ec2Client.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{
					Filter: []ec2types.Filter{
						{
							Name:   aws.String("vpc-id"),
							Values: []string{vpcId},
						},
					},
				})
				Expect(err).To(BeNil())
				Expect(natOut.NatGateways).NotTo(BeEmpty())
				for _, nat := range natOut.NatGateways {
					Expect(nat.State).To(Equal(ec2types.NatGatewayStateAvailable))
				}
				By("Check Cloud Provider Resource: EKS Status")
				eksOut, err := eksClient.DescribeCluster(ctx, &eks.DescribeClusterInput{
					Name: aws.String(eksCluster.Name),
				})
				Expect(err).To(Succeed())
				Expect(eksOut.Cluster.Status).To(Equal(ekstypes.ClusterStatusActive))
				By("Check Cloud Provider: NodeGroup Status")
				ngs, err := eksClient.ListNodegroups(ctx, &eks.ListNodegroupsInput{
					ClusterName: aws.String(eksCluster.Name),
				})
				Expect(err).To(Succeed())
				Expect(ngs.Nodegroups).NotTo(BeEmpty())
			case "gcp":
				By("Call QueryVpcClusterIds API")
				gcpInfo := out.Vpcs[0].GetGcp()
				Expect(gcpInfo).NotTo(BeNil())
				in := new(regionalpb.QueryVpcClusterIdsRequest)
				in.TenantId = c.Cluster.TenantId
				in.ProjectId = c.Cluster.ProjectId
				in.Info = &regionalpb.QueryVpcClusterIdsRequest_Gcp{
					Gcp: &regionalpb.Vpc_GcpVpcInfo{
						ProjectId:   gcpInfo.GetProjectId(),
						NetworkName: gcpInfo.GetNetworkName(),
					},
				}
				out, err := vpcClient.QueryVpcClusterIds(ctx, in)
				Expect(err).To(BeNil())
				Expect(out.ClusterIds).To(HaveLen(1))
				Expect(out.ClusterIds[0]).To(Equal(c.Cluster.Id))
				By("Check Crossplane CR: GKE Status")
				gkeCluster := new(gkev1beta2.Cluster)
				gkeCluster.Name = namer.UniqueIdentifier(appboxCR.ObjectMeta, namer.WithLimit(gke.GKENameLimit))
				Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster)).To(Succeed())
				ready, synced := utils.GetConditions(gkeCluster.Status.Conditions)
				Expect(ready && synced).To(BeTrue())
				By("Check Crossplane CR: NodeGroup Status")
				np := new(gkev1beta1.NodePool)
				np.Name = namer.UniqueIdentifier(
					appboxCR.ObjectMeta,
					namer.WithPrefix(consts.TiDBCloudPrefix),
					namer.WithLimit(gke.GKENameLimit),
				)
				Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(np), np)).To(Succeed())
				ready, synced = utils.GetConditions(np.Status.Conditions)
				Expect(ready && synced).To(BeTrue())

				By("Check Crossplane CR: Router Status")
				router := new(computev1alpha1.Router)
				router.Name = namer.UniqueIdentifier(internetCR.ObjectMeta, namer.WithPrefix("egress"))
				Expect(kubeClient.Get(ctx, client.ObjectKeyFromObject(router), router)).To(Succeed())
				ready, synced = utils.GetConditions(router.Status.Conditions)
				Expect(ready && synced).To(BeTrue())

				By("Check Cloud Provider Resource: Network Status")
				netOut, err := gcpNetworksClient.Get(ctx, &computepb.GetNetworkRequest{
					Network: vpcId,
					Project: f.Config.DefaultAccount,
				})
				Expect(err).To(BeNil())
				Expect(netOut.Name).To(Equal(vpcId))
				Expect(netOut.Subnetworks).NotTo(BeEmpty())
				By("Check Cloud Provider Resource: Router Status")
				routerOut, err := gcpRoutersClient.Get(ctx, &computepb.GetRouterRequest{
					Project: f.Config.DefaultAccount,
					Region:  f.Config.Region,
					Router:  router.Name,
				})
				Expect(err).To(BeNil())
				Expect(*routerOut.Network).To(Equal(vpcId))
				Expect(routerOut.Nats).NotTo(BeEmpty())

				By("Check Cloud Provider Resource: GKE Status")
				gkeOut, err := gcpContainerClient.GetCluster(ctx, &containerpb.GetClusterRequest{
					Name: fmt.Sprintf("projects/%s/locations/%s/clusters/%s", f.Config.DefaultAccount, f.Config.Region, gkeCluster.Name),
				})
				Expect(err).To(Succeed())
				Expect(gkeOut.Network).To(Equal(vpcId))
				Expect(gkeOut.NodePools).NotTo(BeEmpty())
				for _, p := range gkeOut.NodePools {
					Expect(p.Status).To(Equal(containerpb.NodePool_RUNNING))
				}
			}

		})

		It("Should return 409 when create Cluster with Same Id", func() {
			_, err = clusterClient.CreateCluster(ctx, c)
			Expect(err).Should(HaveOccurred())
			Expect(err.Error()).To(ContainSubstring("HTTPSTATUSCODE:409"))
		})

		It("Should Create APPRoute Only When Creating Another Cluster With Same vpcId For Same Tenant,Project", func() {
			cReq := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cReq)
			Expect(err).To(Succeed())
			appRouteCR := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cReq.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRouteCR, 5*time.Minute, interval)
			Expect(appRouteCR.Spec.APPBoxRef.Name).To(Equal(fmt.Sprintf(vpcName + "-0")))
		})

		It("Should Create APPRoute Only When Creating Another Cluster With Same vpcId For Different Tenant,Project", func() {
			By("Create Cluster with different tenant and project")
			cBarBar := CreateClusterReq(f.TenantBar, f.ProjectBar, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cBarBar)
			Expect(err).To(Succeed())
			appRouteBarBar := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cBarBar.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRouteBarBar, 5*time.Minute, interval)
			Expect(appRouteBarBar.Spec.APPBoxRef.Name).To(Equal(fmt.Sprintf(vpcName + "-0")))

			By("Create Cluster with same tenant but different project")
			cFooBar := CreateClusterReq(f.TenantFoo, f.ProjectBar, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cFooBar)
			Expect(err).To(Succeed())
			appRouteFooBar := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cFooBar.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRouteFooBar, 5*time.Minute, interval)
			Expect(appRouteFooBar.Spec.APPBoxRef.Name).To(Equal(fmt.Sprintf(vpcName + "-0")))
		})

	})

	It("Should Return Error When VPC is Deleting or not exists", func() {
		By("Create Cluster when VPC is not exist")
		c := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, Nonexistent)
		_, err := clusterClient.CreateCluster(ctx, c)
		Expect(err).Should(HaveOccurred())
		Expect(err.Error()).To(ContainSubstring("HTTPSTATUSCODE:400"))

		By("Create Cluster When VPC is Deleting")
		req := CreateVPCReq(f.TenantFoo, f.ProjectFoo)
		_, err = vpcClient.CreateVpc(ctx, req)
		Expect(err).To(BeNil())
		vpcCR := &rsv1alpha1.VPC{
			ObjectMeta: metav1.ObjectMeta{
				Name:      utils.MD5(req.RequestId),
				Namespace: f.Config.Namespace,
			},
		}
		waitVPCReady(ctx, vpcCR, 5*time.Minute, interval)
		vpcId := vpcCR.Status.VPC.VPCID
		vpcClient.DeleteVpc(ctx, &regionalpb.DeleteVpcRequest{VpcId: vpcId})
		waitVPCToPhase(ctx, vpcCR, rsv1alpha1.VPCPhaseDeleting, 10*time.Minute, 5*time.Second)
		c = CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
		_, err = clusterClient.CreateCluster(ctx, c)
		Expect(err).Should(HaveOccurred())
		Expect(err.Error()).To(ContainSubstring("HTTPSTATUSCODE:400"))
	})

	Context("Create Cluster When Resources is Deleting", func() {
		var vpcName, vpcId string
		var err error
		var vpcCR *rsv1alpha1.VPC
		BeforeEach(func() {
			By("Create and Wait For VPC Ready Within 5 mins")
			req := CreateVPCReq(f.TenantFoo, f.ProjectFoo)
			vpcName = utils.MD5(req.RequestId)
			_, err = vpcClient.CreateVpc(ctx, req)
			Expect(err).To(BeNil())
			vpcCR = &rsv1alpha1.VPC{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName,
					Namespace: f.Config.Namespace,
				},
			}
			waitVPCReady(ctx, vpcCR, 5*time.Minute, interval)
			vpcId = vpcCR.Status.VPC.VPCID

			By("Create and Wait For Cluster Ready Within 30 mins")
			c := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, c)
			Expect(err).To(Succeed())
			appRoute := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      c.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRoute, 30*time.Minute, interval)
		})

		It("Should Create new Internet with increased index When Internet is deleting", func() {
			By("Mock GC Controller Behavior To Delete Internet")
			internet0 := &rsv1alpha1.Internet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			Expect(kubeClient.Delete(ctx, internet0)).To(Succeed())
			waitInternetToPhase(ctx, internet0, rsv1alpha1.InternetPhaseDeleting, 1*time.Minute, 5*time.Second)

			// if internet0 is deleted, return directly.
			if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(internet0), internet0); apierrors.IsNotFound(err) {
				return
			}
			By("Create New Cluster When Internet is Deleting")
			cNew := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cNew)
			Expect(err).To(Succeed())
			internet1 := &rsv1alpha1.Internet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-1",
					Namespace: f.Config.Namespace,
				},
			}
			waitInternetReady(ctx, internet1, 10*time.Minute, interval)
			Expect(internet1.Spec.VPCRef.Name).To(Equal(vpcName))
			appRouteNew := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cNew.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRouteNew, 5*time.Minute, interval)
			Expect(appRouteNew.Spec.APPBoxRef.Name).To(Equal(internet0.Name))
		})

		It("Should Create New APPBox With Increased Index When APPBox is deleting", func() {
			By("Mock GC Controller Behavior To Delete APPBox")
			appBox0 := &rsv1alpha1.APPBox{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			Expect(kubeClient.Delete(ctx, appBox0)).To(Succeed())
			waitAPPBoxToPhase(ctx, appBox0, rsv1alpha1.APPBoxPhaseDeleting, 1*time.Minute, 5*time.Second)
			By("Create New Cluster When APPBox is Deleting")
			cNew := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cNew)
			Expect(err).To(Succeed())
			appBox1 := &rsv1alpha1.APPBox{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-1",
					Namespace: f.Config.Namespace,
				},
			}
			By("Check APPBox-1 ready")
			waitAPPBoxReady(ctx, appBox1, 40*time.Minute, interval)
			Expect(appBox1.Spec.VPCRef).NotTo(BeNil())
			Expect(appBox1.Spec.VPCRef.Name).To(Equal(vpcName))
			appRouteNew := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cNew.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			By("Check APPRoute Ready And Ref to APPBox-1")
			waitAPPRouteReady(ctx, appRouteNew, 5*time.Minute, interval)
			Expect(appRouteNew.Spec.APPBoxRef.Name).To(Equal(appBox1.Name))
		})
	})
})

var _ = Describe("Delete Cluster API Test", Label("DeleteCluster"), func() {
	Context("Resource Deletion And Reuse", Ordered, func() {
		var vpcName, vpcId string
		var appBox *rsv1alpha1.APPBox
		var internet *rsv1alpha1.Internet
		var err error
		It("Should Delete APPRoute Only And Keep Other Resources When Delete Concurrently", func() {
			By("Create and Wait For VPC Ready Within 5 mins")
			req := CreateVPCReq(f.TenantFoo, f.ProjectFoo)
			vpcName = utils.MD5(req.RequestId)
			_, err = vpcClient.CreateVpc(ctx, req)
			Expect(err).To(BeNil())
			vpcCR := &rsv1alpha1.VPC{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName,
					Namespace: f.Config.Namespace,
				},
			}
			waitVPCReady(ctx, vpcCR, 5*time.Minute, interval)
			vpcId = vpcCR.Status.VPC.VPCID
			By("Create and Wait For Cluster Ready Within 30 mins")
			c := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, c)
			Expect(err).To(Succeed())
			appRoute := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      c.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRoute, 30*time.Minute, interval)

			By("Delete Cluster And Check Resources Status")
			errG, _ := errgroup.WithContext(ctx)
			for i := 0; i < 3; i++ {
				errG.Go(func() error {
					_, err = clusterClient.DeleteCluster(ctx, &regionalpb.DeleteClusterRequest{ClusterId: c.Cluster.Id})
					return err
				})
			}
			Expect(errG.Wait()).To(BeNil())
			Eventually(func() bool {
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(appRoute), appRoute)
				return err != nil && client.IgnoreNotFound(err) == nil
			}, 5*time.Minute, interval).Should(BeTrue())

			By("Check APPBox Should Exists")
			appBox = &rsv1alpha1.APPBox{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			err = kubeClient.Get(ctx, client.ObjectKeyFromObject(appBox), appBox)
			Expect(err).To(Succeed())
			Expect(appBox.Status.Phase).To(Equal(rsv1alpha1.APPBoxPhaseActive))
			By("Check Internet Should Exists")
			internet = &rsv1alpha1.Internet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			err = kubeClient.Get(ctx, client.ObjectKeyFromObject(internet), internet)
			Expect(err).To(Succeed())
			Expect(internet.Status.Phase).To(Equal(rsv1alpha1.InternetPhaseActive))
		})

		It("Should Reuse APPBox if APPBox is idle When Creating Another Cluster Under Same VPC", func() {
			err = kubeClient.Get(ctx, client.ObjectKeyFromObject(appBox), appBox)
			Expect(err).To(Succeed())
			By("Mock GC Controller Behavior To Update APPBox Status As ReadyForDelete")
			appBox.Status.Phase = rsv1alpha1.APPBoxPhaseReadyForDelete
			err = kubeClient.Status().Update(ctx, appBox)
			Expect(err).To(Succeed())
			waitAPPBoxToPhase(ctx, appBox, rsv1alpha1.APPBoxPhaseReadyForDelete, 1*time.Minute, interval)
			By("Create Another Cluster Under Same VPC")
			cReq := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cReq)
			Expect(err).To(Succeed())

			By("Check APPBox Reused And Status Changed To Active")
			appRoute := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cReq.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRoute, 5*time.Minute, interval)
			Expect(appRoute.Spec.APPBoxRef.Name).To(Equal(appBox.Name))
			waitAPPBoxReady(ctx, appBox, 5*time.Minute, interval)
		})

		It("Should Reuse Internet if Internet is idle When Creating Cluster Under Same VPC", func() {
			err = kubeClient.Get(ctx, client.ObjectKeyFromObject(internet), internet)
			Expect(err).To(Succeed())
			By("Mock GC Controller Behavior To Update Internet Status As ReadyForDelete")
			internet.Status.Phase = rsv1alpha1.InternetPhaseReadyForDelete
			err = kubeClient.Status().Update(ctx, internet)
			Expect(err).To(Succeed())
			cReq := CreateClusterReq(f.TenantFoo, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, cReq)
			Expect(err).To(Succeed())
			By("Check Internet Reused And Status Changed To Active")
			appRoute := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      cReq.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRoute, 5*time.Minute, interval)
			waitInternetReady(ctx, internet, 5*time.Minute, interval)
		})

	})

	It("Should Return 200 When Delete Cluster That Is Not Existed", func() {
		_, err := clusterClient.DeleteCluster(ctx, &regionalpb.DeleteClusterRequest{ClusterId: Nonexistent})
		Expect(err).To(Succeed())
	})

	It("Should Delete All Resources After Delete Cluster And GCed for all accounts", Label("XAccount"), func() {
		for i, tenant := range f.TenantAcc {
			account := f.Config.AccountIDList[i]
			By(fmt.Sprintf("Run Test For Tenant %s Binds To Account %s", tenant, account))
			By("Create And Wait VPC Ready Within 5 mins")
			req := CreateVPCReq(tenant, f.ProjectFoo)
			vpcName := utils.MD5(req.RequestId)
			_, err := vpcClient.CreateVpc(ctx, req)
			Expect(err).To(BeNil())
			vpcCR := &rsv1alpha1.VPC{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName,
					Namespace: f.Config.Namespace,
				},
			}
			waitVPCReady(ctx, vpcCR, 5*time.Minute, interval)
			vpcId := vpcCR.Status.VPC.VPCID

			By("Create and Wait For Cluster Ready Within 30 mins")
			c := CreateClusterReq(tenant, f.ProjectFoo, f.Config.Provider, vpcId)
			_, err = clusterClient.CreateCluster(ctx, c)
			Expect(err).To(Succeed())
			appRoute := &rsv1alpha1.APPRoute{
				ObjectMeta: metav1.ObjectMeta{
					Name:      c.Cluster.Id,
					Namespace: f.Config.Namespace,
				},
			}
			waitAPPRouteReady(ctx, appRoute, 30*time.Minute, interval)

			By("Delete Cluster And Check APPRoute Deleted")
			_, err = clusterClient.DeleteCluster(ctx, &regionalpb.DeleteClusterRequest{ClusterId: c.Cluster.Id})
			Expect(err).To(Succeed())
			Eventually(func() bool {
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(appRoute), appRoute)
				return err != nil && client.IgnoreNotFound(err) == nil
			}, 5*time.Minute, interval).Should(BeTrue())

			By("Check APPBox Deleted")
			appBox := &rsv1alpha1.APPBox{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			Eventually(func() bool {
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(appBox), appBox)
				return err != nil && client.IgnoreNotFound(err) == nil
			}, 30*time.Minute, interval).Should(BeTrue())

			By("Check Internet Deleted")
			internet := &rsv1alpha1.Internet{
				ObjectMeta: metav1.ObjectMeta{
					Name:      vpcName + "-0",
					Namespace: f.Config.Namespace,
				},
			}
			Eventually(func() bool {
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(internet), internet)
				return err != nil && client.IgnoreNotFound(err) == nil
			}, 30*time.Minute, interval).Should(BeTrue())

			switch f.Config.Provider {
			case "aws":
				eksCliForAcc := awsClient.EKS(GetAssumedRole(account, f.Config.RoleName))
				ec2CliForAcc := awsClient.EC2(GetAssumedRole(account, f.Config.RoleName))
				By("Check Crossplane EKS Deleted")
				eksCluster := new(eksv1beta1.Cluster)
				eksCluster.Name = namer.UniqueIdentifier(appBox.ObjectMeta)
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(eksCluster), eksCluster)
				Expect(err != nil && client.IgnoreNotFound(err) == nil).To(BeTrue())
				By("Check Crossplane NodeGroup Deleted")
				ng := new(manualv1alpha1.NodeGroup)
				ng.Name = namer.UniqueIdentifier(appBox.ObjectMeta, namer.WithPrefix(consts.TiDBCloudPrefix))
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(ng), ng)
				Expect(err != nil && client.IgnoreNotFound(err) == nil).To(BeTrue())
				By("Check Crossplane IGW Deleted")
				igw := new(ec2v1beta1.InternetGateway)
				igw.Name = namer.UniqueIdentifier(internet.ObjectMeta)
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(igw), igw)
				Expect(err != nil && client.IgnoreNotFound(err) == nil).To(BeTrue())

				By("Check Cloud Provider: VPC Status")
				vpcOut, err := ec2CliForAcc.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{
					VpcIds: []string{vpcId},
				})
				Expect(err).To(BeNil())
				Expect(vpcOut.Vpcs).To(HaveLen(1))
				Expect(vpcOut.Vpcs[0].State).To(Equal(ec2types.VpcStateAvailable))
				By("Check Cloud Provider: IGW Status")
				igwOut, err := ec2CliForAcc.DescribeInternetGateways(ctx, &ec2.DescribeInternetGatewaysInput{
					Filters: []ec2types.Filter{
						{
							Name:   aws.String("attachment.vpc-id"),
							Values: []string{vpcId},
						},
					},
				})
				Expect(err).To(BeNil())
				Expect(igwOut.InternetGateways).To(BeEmpty())
				By("Check Cloud Provider: NAT Status")
				natOut, err := ec2CliForAcc.DescribeNatGateways(ctx, &ec2.DescribeNatGatewaysInput{
					Filter: []ec2types.Filter{
						{
							Name:   aws.String("vpc-id"),
							Values: []string{vpcId},
						},
					},
				})
				Expect(err).To(BeNil())
				Expect(natOut.NatGateways).To(BeEmpty())
				By("Check Cloud Provider: EKS Status")
				eksOut, err := eksCliForAcc.DescribeCluster(ctx, &eks.DescribeClusterInput{
					Name: aws.String(eksCluster.Name),
				})
				Expect(err).To(Succeed())
				Expect(eksOut.Cluster).To(BeNil())
				By("Check Cloud Provider: Nodegroup Status")
				ngs, err := eksCliForAcc.ListNodegroups(ctx, &eks.ListNodegroupsInput{
					ClusterName: aws.String(eksCluster.Name),
				})
				Expect(err).To(Succeed())
				Expect(ngs).NotTo(BeEmpty())
			case "gcp":
				By("Check Crossplane GKE Deleted")
				gkeCluster := new(gkev1beta2.Cluster)
				gkeCluster.Name = namer.UniqueIdentifier(appBox.ObjectMeta, namer.WithLimit(gke.GKENameLimit))
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(gkeCluster), gkeCluster)
				Expect(err != nil && client.IgnoreNotFound(err) == nil).To(BeTrue())
				By("Check Crossplane NodePool Deleted")
				np := new(gkev1beta1.NodePool)
				np.Name = namer.UniqueIdentifier(
					appBox.ObjectMeta,
					namer.WithPrefix(consts.TiDBCloudPrefix),
					namer.WithLimit(gke.GKENameLimit),
				)
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(np), np)
				Expect(err != nil && client.IgnoreNotFound(err) == nil).To(BeTrue())
				By("Check Crossplane Router Deleted")
				router := new(computev1alpha1.Router)
				router.Name = namer.UniqueIdentifier(internet.ObjectMeta, namer.WithPrefix("egress"))
				err = kubeClient.Get(ctx, client.ObjectKeyFromObject(router), router)
				Expect(err != nil && client.IgnoreNotFound(err) == nil).To(BeTrue())
				By("Check Network Status")
				netOut, err := gcpNetworksClient.Get(ctx, &computepb.GetNetworkRequest{
					Network: vpcId,
					Project: account,
				})
				Expect(err).To(BeNil())
				Expect(netOut.Subnetworks).NotTo(BeEmpty())
				By("Check Router Status")
				var gapierr *googleapi.Error
				_, err = gcpRoutersClient.Get(ctx, &computepb.GetRouterRequest{
					Project: account,
					Region:  f.Config.Region,
					Router:  router.Name,
				})
				Expect(errors.As(err, &gapierr)).To(BeTrue())
				Expect(gapierr.Code).To(Equal(GNotFound))

				By("Check GKE Status")
				_, err = gcpContainerClient.GetCluster(ctx, &containerpb.GetClusterRequest{
					Name: fmt.Sprintf("projects/%s/locations/%s/clusters/%s", account, f.Config.Region, gkeCluster.Name),
				})
				Expect(errors.As(err, &gapierr)).To(BeTrue())
				Expect(gapierr.Code).To(Equal(GNotFound))
			}
		}
	})
})

func CreateVPCReq(tenant, project string) *regionalpb.CreateVpcRequest {
	return &regionalpb.CreateVpcRequest{
		RequestId: fmt.Sprintf("%d", time.Now().UnixNano()),
		Vpc: &regionalpb.Vpc{
			TenantIds:  []string{tenant},
			ProjectIds: []string{project},
			CidrBlock:  "10.0.0.0/16",
		},
	}
}

func CreateClusterReq(tenant, project, provider, vpcId string) *regionalpb.CreateClusterRequest {
	cluster := &regionalpb.Cluster{
		Id:        fmt.Sprintf("%d", time.Now().UnixNano()),
		TenantId:  tenant,
		ProjectId: project,
		ComponentNodeSpecs: []*regionalpb.Cluster_NodeSpec{
			{
				ComponentType: regionalpb.ComponentType_PD,
				InstanceType:  "foo",
				Count:         1,
			},
		},
	}

	switch provider {
	case string(constants.AWS):
		cluster.VpcSettings = &regionalpb.Cluster_VpcSettings{
			Vpc: &regionalpb.Cluster_VpcSettings_ExistingAwsVpcId{
				ExistingAwsVpcId: vpcId,
			},
		}
	case string(constants.GCP):
		cluster.VpcSettings = &regionalpb.Cluster_VpcSettings{
			Vpc: &regionalpb.Cluster_VpcSettings_ExistingGcpNetworkName{
				ExistingGcpNetworkName: vpcId,
			},
		}
	}

	return &regionalpb.CreateClusterRequest{Cluster: cluster}
}

func waitVPCReady(ctx context.Context, vpc *rsv1alpha1.VPC, timeout, interval time.Duration) {
	waitVPCToPhase(ctx, vpc, rsv1alpha1.VPCPhaseActive, timeout, interval)
}

func waitVPCToPhase(ctx context.Context, vpc *rsv1alpha1.VPC, phase rsv1alpha1.VPCPhase, timeout, interval time.Duration) {
	Eventually(func() error {
		key := client.ObjectKeyFromObject(vpc)
		err := kubeClient.Get(ctx, key, vpc)
		if err != nil {
			logger.Error(err, "observing vpc status", "vpc", key.String())
			return err
		}
		if vpc.Status.Phase != phase {
			logger.Info("wait vpc phase sync", "vpc", vpc.Name, "expect", phase, "current ", vpc.Status.Phase)
			return errWait
		}
		return nil
	}, timeout, interval).Should(Succeed(), fmt.Sprintf("vpc should be %s", phase))
}

func waitAPPBoxReady(ctx context.Context, appbox *rsv1alpha1.APPBox, timeout, interval time.Duration) {
	waitAPPBoxToPhase(ctx, appbox, rsv1alpha1.APPBoxPhaseActive, timeout, interval)
}

func waitAPPBoxToPhase(ctx context.Context, appbox *rsv1alpha1.APPBox, phase rsv1alpha1.APPBoxPhase, timeout, interval time.Duration) {
	Eventually(func() error {
		err := kubeClient.Get(ctx, client.ObjectKeyFromObject(appbox), appbox)

		if err != nil {
			logger.Error(err, "observing appbox status", "appbox", appbox.Name)
			return err
		}
		if appbox.Status.Phase != phase {
			logger.Info("wait appbox phase sync", "appbox", appbox.Name, "expect", phase, "current ", appbox.Status.Phase)
			return errWait
		}
		return nil
	}, timeout, interval).Should(Succeed(), fmt.Sprintf("appBox should be %s", phase))
}

func waitAPPRouteReady(ctx context.Context, approute *rsv1alpha1.APPRoute, timeout, interval time.Duration) {
	waitAPPRouteToPhase(ctx, approute, rsv1alpha1.APPRoutePhaseActive, timeout, interval)
}

func waitAPPRouteToPhase(ctx context.Context, approute *rsv1alpha1.APPRoute, phase rsv1alpha1.APPRoutePhase, timeout, interval time.Duration) {
	Eventually(func() error {
		err := kubeClient.Get(ctx, client.ObjectKeyFromObject(approute), approute)

		if err != nil {
			logger.Error(err, "observing approute status", "approute", approute.Name)
			return err
		}
		if approute.Status.Phase != phase {
			logger.Info("wait approute phase sync", "approute", approute.Name, "expect", phase, "current", approute.Status.Phase)
			return errWait
		}
		return nil
	}, timeout, interval).Should(Succeed(), fmt.Sprintf("approute should be %s", phase))
}

func waitInternetReady(ctx context.Context, internet *rsv1alpha1.Internet, timeout, interval time.Duration) {
	waitInternetToPhase(ctx, internet, rsv1alpha1.InternetPhaseActive, timeout, interval)
}

func waitInternetToPhase(ctx context.Context, internet *rsv1alpha1.Internet, phase rsv1alpha1.InternetPhase, timeout, interval time.Duration) {
	Eventually(func() error {
		err := kubeClient.Get(ctx, client.ObjectKeyFromObject(internet), internet)

		if err != nil {
			logger.Error(err, "observing internet status", "internet", internet.Name)
			return client.IgnoreNotFound(err)
		}
		if internet.Status.Phase != phase {
			logger.Info("wait internet phase sync", "internet", internet.Name, "expect", phase, "current", internet.Status.Phase)
			return errWait
		}
		return nil
	}, timeout, interval).Should(Succeed(), fmt.Sprintf("internet should be %s", phase))
}