// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.package spec

package controller

import (
	"fmt"
	"time"

	"github.com/ngaut/log"
	"github.com/pingcap/tidb-operator/pkg/apis/pingcap.com/v1"
	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
	"github.com/pingcap/tidb-operator/pkg/review"
	"github.com/pingcap/tidb-operator/pkg/util/errors"
	"github.com/pingcap/tidb-operator/pkg/util/label"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
)

var (
	keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc
	// ControllerKind contains the schema.GroupVersionKind for this controller type
	ControllerKind = v1.SchemeGroupVersion.WithKind("TidbCluster")
	// RBACEnabled is set at startup by the main program
	RBACEnabled    bool
	propBackGround = metav1.DeletePropagationBackground
	deleteOptions  = &metav1.DeleteOptions{PropagationPolicy: &propBackGround}
)

// TiDBClusterController is a controller for managing TidbCluster.
// It manages tidb/pd service, tidbcluster config and tidbcluster itself
type TiDBClusterController struct {
	cli        versioned.Interface
	kubeCli    kubernetes.Interface
	controller cache.Controller
	store      cache.Store
	queue      workqueue.RateLimitingInterface
}

// NewTiDBClusterController returns a *TiDBClusterController
func NewTiDBClusterController(cli versioned.Interface, kubeCli kubernetes.Interface, resyncDuration time.Duration) *TiDBClusterController {
	tcc := &TiDBClusterController{
		cli:     cli,
		kubeCli: kubeCli,
		queue:   workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "tidbclustercontroller"),
	}
	tcc.store, tcc.controller = cache.NewInformer(
		&cache.ListWatch{
			ListFunc: cache.ListFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
				return tcc.cli.PingcapV1().TidbClusters(metav1.NamespaceAll).List(opts)
			}),
			WatchFunc: cache.WatchFunc(func(opts metav1.ListOptions) (watch.Interface, error) {
				return tcc.cli.PingcapV1().TidbClusters(metav1.NamespaceAll).Watch(opts)
			}),
		},
		&v1.TidbCluster{},
		resyncDuration,
		cache.ResourceEventHandlerFuncs{
			AddFunc:    tcc.enqueueTidbCluster,
			UpdateFunc: tcc.updateTidbCluster,
			DeleteFunc: tcc.enqueueTidbCluster,
		})
	return tcc
}

// Run start the controller.
func (tcc *TiDBClusterController) Run(workers int, stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer tcc.queue.ShutDown()
	log.Infof("Starting TidbCluster controller")
	go tcc.controller.Run(wait.NeverStop)

	for i := 0; i < workers; i++ {
		go wait.Until(tcc.worker, time.Second, stopCh)
	}

	<-stopCh
	log.Infof("Shutting down TidbCluster controller")
}

func (tcc *TiDBClusterController) enqueueTidbCluster(obj interface{}) {
	key, err := keyFunc(obj)
	if err != nil {
		log.Errorf("can't get key for obj: %v, error: %v", obj, err)
		return
	}

	tcc.queue.Add(key)
}

func (tcc *TiDBClusterController) worker() {
	for {
		func() {
			key, quit := tcc.queue.Get()
			if quit {
				return
			}
			defer tcc.queue.Done(key)
			if err := tcc.syncTidbCluster(key.(string)); err != nil {
				utilruntime.HandleError(fmt.Errorf("Error syncing TidbCluster %v, requeuing: %v", key.(string), err))
				tcc.queue.AddRateLimited(key)
			} else {
				tcc.queue.Forget(key)
			}
		}()
	}
}

func (tcc *TiDBClusterController) syncTidbCluster(key string) error {
	startTime := time.Now()
	defer func() {
		log.Infof("Finished syncing TidbCluster[%s] (%v)", key, time.Since(startTime))
	}()

	obj, exists, getErr := tcc.store.GetByKey(key)
	if getErr != nil {
		return errors.Logf(getErr, "try to get obj: %s", key)
	}

	ns, clusterName, splitErr := cache.SplitMetaNamespaceKey(key)
	if splitErr != nil {
		return errors.Logf(splitErr, "error splitting TidbCluster key: %q", key)
	}

	if !exists {
		log.Infof("TidbCluster [%s] doesn't exist in store", key)
		// make sure tidbcluster is actually deleted
		cliV1 := tcc.cli.PingcapV1()
		_, clusterErr := cliV1.TidbClusters(ns).Get(clusterName, metav1.GetOptions{})
		if clusterErr == nil {
			return nil
		}
		if !apierrors.IsNotFound(clusterErr) {
			return clusterErr
		}
		if apierrors.IsNotFound(clusterErr) {
			msg := fmt.Sprintf("TidbCluster: [%s/%s] was deleted, deleting all releated resources", ns, clusterName)
			log.Info(msg)

			ts, setErr := cliV1.TidbSets(ns).Get(fmt.Sprintf("%s-pdset", clusterName), metav1.GetOptions{})
			if setErr != nil {
				return errors.Trace(setErr)
			}
			uid := string(ts.UID)
			if len(ts.OwnerReferences) > 0 {
				or := ts.OwnerReferences[0]
				uid = string(or.UID)
			}

			err := review.Review(tcc.cli, review.Info{
				UID:         uid,
				Namespace:   ns,
				ClusterName: clusterName,
				Reason:      "DeleteTidbClusterResources",
				Message:     msg,
			})
			if err != nil {
				return errors.Trace(err)
			}

			return tcc.tryToDeleteTidbClusterResource(ns, clusterName)
		}
	}

	tc, ok := obj.(*v1.TidbCluster)
	if !ok {
		return errors.Errorf("obj: %v is not a *TidbCluster", obj)
	}

	tc, synchErr := tcc.syncClusterStatusFor(tc)
	if synchErr != nil {
		return errors.Logf(synchErr, "failed to sync clusterStatus for key %s", key)
	}

	if tc.ReallyRestoring() {
		log.Infof("TidbCluster: [%s/%s] is restoring", ns, clusterName)

		err := tcc.tryToFinishRetoring(tc)
		if err != nil {
			log.Warnf("restoring not finished: %v", err)
		}
	}

	if tc.ReallyGracefulDeleted() {
		msg := fmt.Sprintf("TidbCluster: [%s/%s] was graceful deleted, deleting all releated resources", ns, clusterName)
		log.Info(msg)

		ts, setErr := tcc.cli.PingcapV1().TidbSets(ns).Get(fmt.Sprintf("%s-pdset", clusterName), metav1.GetOptions{})
		if setErr != nil {
			return errors.Trace(setErr)
		}

		uid := string(ts.UID)
		if len(ts.OwnerReferences) > 0 {
			or := ts.OwnerReferences[0]
			uid = string(or.UID)
		}

		err := review.Review(tcc.cli, review.Info{
			UID:         uid,
			Namespace:   ns,
			ClusterName: clusterName,
			Reason:      "GracefulDeleteTidbCluster",
			Message:     msg,
		})
		if err != nil {
			return errors.Trace(err)
		}

		return tcc.tryToDeleteTidbClusterResource(ns, clusterName)
	}

	if err := tcc.syncConfigMapFor(tc); err != nil {
		return errors.Logf(err, "failed to sync ConfigMap[%s/%s]", ns, clusterName)
	}
	if err := tcc.syncServiceFor(tc); err != nil {
		return errors.Logf(err, "failed to sync Service[%s/%s]", ns, clusterName)
	}
	if err := tcc.syncTidbSetFor(tc); err != nil {
		return errors.Logf(err, "failed to sync TidbSet[%s/%s]", ns, clusterName)
	}
	if err := tcc.syncUpgradeStatusesFor(tc); err != nil {
		return errors.Trace(err)
	}

	if tc.Spec.Monitor != nil {
		if err := tcc.syncMonitorFor(tc); err != nil {
			return errors.Logf(err, "failed to sync Monitor[%s/%s]", ns, clusterName)
		}
	}

	if tc.Spec.PrivilegedTiDB != nil {
		if err := tcc.syncPrivilegedTiDBFor(tc); err != nil {
			return errors.Logf(err, "failed to sync privileged-tidb[%s/%s]", ns, clusterName)
		}
	}

	return nil
}

// TODO: determine whether it's necessary to update tidbcluster
func (tcc *TiDBClusterController) updateTidbCluster(old, cur interface{}) {
	tcc.enqueueTidbCluster(cur)
}

func (tcc *TiDBClusterController) tryToFinishRetoring(tc *v1.TidbCluster) error {
	ns := tc.GetNamespace()
	clusterName := tc.GetName()

	var err error
	if tc.Spec.Config != nil {
		_, err = tcc.kubeCli.CoreV1().ConfigMaps(ns).Get(tc.GetConfigMapName(), metav1.GetOptions{})
		if err != nil {
			return errors.Trace(err)

		}
	}

	_, err = tcc.kubeCli.CoreV1().Services(ns).Get(pdSvcName(clusterName), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	_, err = tcc.kubeCli.CoreV1().Services(ns).Get(pdPeerSvcName(clusterName), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	_, err = tcc.kubeCli.CoreV1().Services(ns).Get(tidbSvcName(clusterName), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	_, err = tcc.cli.PingcapV1().TidbSets(ns).Get(fmt.Sprintf("%s-pdset", clusterName), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	_, err = tcc.cli.PingcapV1().TidbSets(ns).Get(fmt.Sprintf("%s-tidbset", clusterName), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	_, err = tcc.cli.PingcapV1().TidbSets(ns).Get(fmt.Sprintf("%s-tikvset", clusterName), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	err = tcc.ensurePodsRunning(tc, label.New().Cluster(clusterName).PD())
	if err != nil {
		return errors.Trace(err)
	}

	err = tcc.ensurePodsRunning(tc, label.New().Cluster(clusterName).TiDB())
	if err != nil {
		return errors.Trace(err)
	}

	err = tcc.ensurePodsRunning(tc, label.New().Cluster(clusterName).TiKV())
	if err != nil {
		return errors.Trace(err)
	}

	tc.Spec.State = v1.StateNormal
	tc.Status.NormalStatus.ReallyNormal = true

	tc.Status.RestoreStatus.EndTime = &metav1.Time{Time: time.Now()}
	tc.Status.RestoreStatus.ReallyRestoring = false
	_, err = tcc.cli.PingcapV1().TidbClusters(ns).Update(tc)
	if err != nil {
		log.Errorf("TidbCluster: [%s/%s] restored failed, %v", ns, clusterName, err)
		return errors.Trace(err)
	}

	log.Infof("TidbCluster: [%s/%s] restored successfully", ns, clusterName)
	return nil
}

func (tcc *TiDBClusterController) ensurePodsRunning(tc *v1.TidbCluster, l label.Label) error {
	ns := tc.GetNamespace()
	clusterName := tc.GetName()

	podList, err := tcc.kubeCli.CoreV1().Pods(ns).List(metav1.ListOptions{LabelSelector: labels.SelectorFromSet(l.Labels()).String()})
	if err != nil {
		return errors.Trace(err)
	}

	podCount := len(podList.Items)
	var size int32
	switch l.AppType() {
	case label.PDLabelVal:
		size = tc.Spec.PD.Size
	case label.TiDBLabelVal:
		size = tc.Spec.TiDB.Size
	case label.TiKVLabelVal:
		size = tc.Spec.TiKV.Size
	}

	if podCount != int(size) {
		return errors.Errorf("TidbCluster: [%s/%s]'s %s size is: [%d], but Pod's count is: [%d]", ns, clusterName, l.AppType(), size, podCount)
	}

	tidbset, err := tcc.cli.PingcapV1().TidbSets(ns).Get(fmt.Sprintf("%s-%sset", clusterName, l.AppType()), metav1.GetOptions{})
	if err != nil {
		return errors.Trace(err)
	}

	for _, pod := range podList.Items {
		if !tidbset.IsReady(&pod) {
			return errors.Errorf("TidbCluster: [%s/%s]'s %s Pod: [%s/%s] is not ready", ns, clusterName, l.AppType(), ns, pod.GetName())
		}
	}

	return nil
}

func (tcc *TiDBClusterController) updateTC(tc *v1.TidbCluster) (*v1.TidbCluster, error) {
	ns := tc.GetNamespace()
	name := tc.GetName()

	tc, err := tcc.cli.PingcapV1().TidbClusters(ns).Update(tc)
	if err != nil {
		log.Errorf("failed to update TidbCluster: [%s/%s], %v", ns, name, err)
		return tc, errors.Trace(err)
	}

	log.Infof("update TidbCluster: [%s/%s] successfully", ns, name)
	return tc, nil
}

// TODO add e2e specs to ensure this
func (tcc *TiDBClusterController) syncClusterStatusFor(tc *v1.TidbCluster) (*v1.TidbCluster, error) {
	ns := tc.GetNamespace()
	name := tc.GetName()

	if tc.Spec.State == v1.StateNormal && !tc.ReallyNormal() && !tc.ReallyGracefulDeleted() && !tc.ReallyRestoring() {
		log.Infof("TidbCluster: [%s/%s]'s Spec.State is: [%s], ReallyNormal is: [%v], ReallyGracefulDeleted is: [%v], ReallyRestoring is: [%v], changing ReallyNormal to true",
			ns, name,
			tc.Spec.State,
			tc.Status.NormalStatus.ReallyNormal,
			tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted,
			tc.Status.RestoreStatus.ReallyRestoring,
		)
		tc.Status.NormalStatus.ReallyNormal = true
		tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted = false
		tc.Status.RestoreStatus.ReallyRestoring = false
		return tcc.updateTC(tc)
	}

	if tc.Spec.State == v1.StateGracefulDeleted && !tc.ReallyGracefulDeleted() && tc.ReallyNormal() && !tc.ReallyRestoring() {
		log.Infof("TidbCluster: [%s/%s]'s Spec.State is: [%s], ReallyNormal is: [%v], ReallyGracefulDeleted is: [%v], ReallyRestoring is: [%v], changing ReallyGracefulDeleted to true",
			ns, name,
			tc.Spec.State,
			tc.Status.NormalStatus.ReallyNormal,
			tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted,
			tc.Status.RestoreStatus.ReallyRestoring,
		)
		tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted = true
		tc.Status.NormalStatus.ReallyNormal = false
		tc.Status.RestoreStatus.ReallyRestoring = false
		return tcc.updateTC(tc)
	}

	if tc.Spec.State == v1.StateRestore && !tc.ReallyRestoring() && tc.ReallyGracefulDeleted() && !tc.ReallyNormal() {
		log.Infof("TidbCluster: [%s/%s]'s Spec.State is: [%s], ReallyNormal is: [%v], ReallyGracefulDeleted is: [%v], ReallyRestoring is: [%v], changing ReallyRestoring to true",
			ns, name,
			tc.Spec.State,
			tc.Status.NormalStatus.ReallyNormal,
			tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted,
			tc.Status.RestoreStatus.ReallyRestoring,
		)
		tc.Status.RestoreStatus.ReallyRestoring = true
		tc.Status.NormalStatus.ReallyNormal = false
		tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted = false
		tc.Status.RestoreStatus.BeginTime = &metav1.Time{Time: time.Now()}
		tc.Status.RestoreStatus.EndTime = nil
		tc.Status.GracefulDeletedStatus.ReallyGracefulDeleted = false
		return tcc.updateTC(tc)
	}

	if tc.ReallyNormal() && tc.Spec.State != v1.StateNormal {
		log.Infof("TidbCluster: [%s/%s] is ReallyNormal, but Spec.State is: [%s], changing Spec.State to StateNormal", ns, name, tc.Spec.State)
		tc.Spec.State = v1.StateNormal
		return tcc.updateTC(tc)
	}

	if tc.ReallyGracefulDeleted() && tc.Spec.State != v1.StateGracefulDeleted {
		log.Infof("TidbCluster: [%s/%s] is ReallyGracefulDeleted, but Spec.State is: [%s], changing Spec.State to StateGracefulDeleted", ns, name, tc.Spec.State)
		tc.Spec.State = v1.StateGracefulDeleted
		return tcc.updateTC(tc)
	}

	if tc.ReallyRestoring() && tc.Spec.State != v1.StateRestore {
		log.Infof("TidbCluster: [%s/%s] is ReallyRestoring, but Spec.State is: [%s], changing Spec.State to StateRestore", ns, name, tc.Spec.State)
		tc.Spec.State = v1.StateRestore
		return tcc.updateTC(tc)
	}

	return tc, nil
}

func (tcc *TiDBClusterController) tryToDeleteTidbSet(ns, name string) error {
	err := tcc.cli.PingcapV1().TidbSets(ns).Delete(name, nil)
	if err != nil && !apierrors.IsNotFound(err) {
		log.Errorf("TidbSet[%s/%s] deleted failed: %v", ns, name, err)
		return errors.Trace(err)
	}
	log.Infof("TidbSet[%s/%s] deleted successfully.", ns, name)
	return nil
}

func (tcc *TiDBClusterController) tryToDeleteService(ns, name string) error {
	svc, err := tcc.kubeCli.CoreV1().Services(ns).Get(name, metav1.GetOptions{})
	if err != nil {
		if apierrors.IsNotFound(err) {
			log.Debugf("Service[%s/%s] doesn't exist, doesn't need to be deleted.", ns, name)
			return nil
		}
		log.Errorf("failed to get Service[%s/%s]: %v", ns, name, err)
		return errors.Trace(err)
	}
	if svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
		// this kind of service is reserved for reuse on cloud
		// and will be deleted in sync controller if its owner TidbCluster is absent
		log.Infof("Service[%s/%s] type is LoadBalancer, skip deleting", ns, name)
		return nil
	}

	if err := tcc.kubeCli.CoreV1().Services(ns).Delete(name, nil); err != nil {
		log.Errorf("Service[%s/%s] deleted failed: %v", ns, name, err)
		return errors.Trace(err)
	}
	log.Infof("Service[%s/%s] deleted successfully.", ns, name)
	return nil
}

func (tcc *TiDBClusterController) tryToDeleteMonitor(ns, name string) error {
	// delete sa, role, rolebinding, service, deployment, job
	if RBACEnabled {
		rbacName := fmt.Sprintf("%s-tidb-prometheus", name)
		if err := tcc.kubeCli.RbacV1beta1().Roles(ns).Delete(rbacName, nil); err != nil {
			if apierrors.IsNotFound(err) {
				log.Infof("Role %s/%s is not exit, no need to delete.", ns, rbacName)
			} else {
				log.Errorf("delete Role %s/%s failed: %v", ns, rbacName, err)
				return errors.Trace(err)
			}
		} else {
			log.Infof("delete Role %s/%s successfully", ns, rbacName)
		}
		if err := tcc.kubeCli.RbacV1beta1().RoleBindings(ns).Delete(rbacName, nil); err != nil {
			if apierrors.IsNotFound(err) {
				log.Infof("RoleBinding %s/%s is not exit,no need to delete", ns, rbacName)
			} else {
				log.Errorf("delete RoleBinding %s/%s failed: %v", ns, rbacName, err)
				return errors.Trace(err)
			}
		} else {
			log.Infof("delete RoleBinding %s/%s successfully", ns, rbacName)
		}
		if err := tcc.kubeCli.CoreV1().ServiceAccounts(ns).Delete(rbacName, nil); err != nil {
			if apierrors.IsNotFound(err) {
				log.Infof("ServiceAccount %s/%s is exit,no need to delete.", ns, rbacName)
			} else {
				log.Errorf("delete ServiceAccount %s/%s failed: %v", ns, rbacName, err)
				return errors.Trace(err)
			}
		} else {
			log.Infof("delete ServiceAccount %s/%s successfully", ns, rbacName)
		}
	}
	services := []string{
		"prometheus",
		"grafana",
	}
	for _, svc := range services {
		svcName := fmt.Sprintf("%s-%s", name, svc)
		if err := tcc.kubeCli.CoreV1().Services(ns).Delete(svcName, nil); err != nil {
			if apierrors.IsNotFound(err) {
				log.Infof("Service %s/%s not exist, no need to delete.", ns, svcName)
			} else {
				log.Errorf("delete Service %s/%s failed: %v", ns, svcName, err)
				return errors.Trace(err)
			}
		} else {
			log.Infof("delete Service %s/%s successfully", ns, svcName)
		}
	}
	deployName := fmt.Sprintf("%s-monitor", name)
	if err := tcc.kubeCli.AppsV1beta1().Deployments(ns).Delete(deployName, deleteOptions); err != nil {
		if apierrors.IsNotFound(err) {
			log.Infof("Deployment %s/%s not exist, no need to delete.", ns, deployName)
		} else {
			log.Errorf("delete Deployment %s/%s failed: %v", ns, deployName, err)
			return errors.Trace(err)
		}
	} else {
		log.Infof("delete Deployment %s/%s successfully", ns, deployName)
	}
	jobName := fmt.Sprintf("%s-configure-grafana", name)
	if err := tcc.kubeCli.BatchV1().Jobs(ns).Delete(jobName, nil); err != nil {
		if apierrors.IsNotFound(err) {
			log.Infof("Job %s/%s not exist, no need to delete.", ns, jobName)
		} else {
			log.Errorf("delete Job %s/%s failed: %v", ns, jobName, err)
			return errors.Trace(err)
		}
	} else {
		log.Infof("delete Job %s/%s successfully", ns, jobName)
	}

	return nil

}

func (tcc *TiDBClusterController) tryToDeletePrivilegedTiDB(ns, name string) error {
	svcName := fmt.Sprintf("%s-privileged-tidb", name)
	if err := tcc.kubeCli.CoreV1().Services(ns).Delete(svcName, nil); err != nil {
		if apierrors.IsNotFound(err) {
			log.Infof("Service %s/%s not exist, no need to delete.", ns, svcName)
		} else {
			log.Errorf("delete Service %s/%s failed: %v", ns, svcName, err)
			return errors.Trace(err)
		}
	} else {
		log.Infof("delete Service %s/%s successfully", ns, svcName)
	}
	deployName := fmt.Sprintf("%s-privileged-tidb", name)
	if err := tcc.kubeCli.AppsV1beta1().Deployments(ns).Delete(deployName, deleteOptions); err != nil {
		if apierrors.IsNotFound(err) {
			log.Infof("Deployment %s/%s not exist, no need to delete", ns, deployName)
		} else {
			log.Errorf("delete deployment %s/%s failed", ns, deployName)
			return errors.Trace(err)
		}
	} else {
		log.Infof("delete Deployment %s/%s successfully", ns, deployName)
	}
	return nil
}

func (tcc *TiDBClusterController) tryToDeleteTidbClusterResource(ns, name string) error {
	pdSetName := fmt.Sprintf("%s-pdset", name)
	tikvSetName := fmt.Sprintf("%s-tikvset", name)
	tidbSetName := fmt.Sprintf("%s-tidbset", name)
	configName := fmt.Sprintf("%s-config", name)

	err := tcc.tryToDeleteTidbSet(ns, tidbSetName)
	if err != nil {
		return errors.Trace(err)
	}
	err = tcc.tryToDeleteTidbSet(ns, tikvSetName)
	if err != nil {
		return errors.Trace(err)
	}
	err = tcc.tryToDeleteTidbSet(ns, pdSetName)
	if err != nil {
		return errors.Trace(err)
	}

	err = tcc.kubeCli.CoreV1().ConfigMaps(ns).Delete(configName, nil)
	if err != nil && !apierrors.IsNotFound(err) {
		log.Errorf("ConfigMap[%s/%s] deleted failed: %v", ns, configName, err)
		return errors.Trace(err)
	}
	log.Infof("ConfigMap[%s/%s] deleted successfully", ns, configName)

	svcNames := []string{pdSvcName(name), pdPeerSvcName(name), tikvPeerSvcName(name), tidbSvcName(name)}
	for _, svcName := range svcNames {
		err = tcc.tryToDeleteService(ns, svcName)
		if err != nil {
			return errors.Trace(err)
		}
	}

	err = tcc.tryToDeleteMonitor(ns, name)
	if err != nil {
		return errors.Trace(err)
	}
	return tcc.tryToDeletePrivilegedTiDB(ns, name)
}