package main

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/spf13/cobra"
	"github.com/tidbcloud/config-center/gitopsfile"
	"github.com/tidbcloud/config-center/gitopsmigrate"
	"github.com/tidbcloud/config-center/pkg/utils"
	pb "github.com/tidbcloud/config-center/proto/config_center"
	"github.com/tidbcloud/core-svc/idl/pbgen/proto/core"
	cloudv1 "github.com/tidbcloud/infra-api/api/v1"
	"go.buf.build/protocolbuffers/go/linuxgit/tidbcloud/base"
	"golang.org/x/mod/semver"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"gopkg.in/yaml.v3"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"
)

// migrateCmd represents the migrate command
var migrateCmd = &cobra.Command{
	Use:   "migrate [gitops path]",
	Short: "migrate from gitops",
	Long:  `migrate from gitops`,
}

func newMigrateBaseCmd() *cobra.Command {
	var gitopsPath, tierPath, minVersion, maxVersion string
	var force bool
	cmd := &cobra.Command{
		Use:   "base",
		Short: "migrate images, default version, config templates and initsql to config-center",
		Long:  `migrate images, default version, config templates and initsql to config-center`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				return err
			}
			defer conn.Close()
			client := pb.NewConfigCenterServiceClient(conn)

			if !force {
				return errors.New("node instance list is only available in SKU api, please use ops sync instead")
			}
			return gitopsfile.ImportBaseFromGitops(client, gitopsPath, tierPath, minVersion, maxVersion, false)
		},
	}

	cmd.Flags().StringVar(&gitopsPath, "gitops", "/gitops-data/tidb-configuration", "local path of tidb-configuration repository")
	cmd.Flags().StringVar(&tierPath, "tier", "/gitops-data/cluster-tier-config", "local path of cluster-tier-config repository")
	cmd.Flags().StringVar(&minVersion, "from", "v6.1.0", "minimum version to migrate")
	cmd.Flags().StringVar(&maxVersion, "to", "v99.99.99", "max version to migrate")
	cmd.Flags().BoolVar(&force, "force", false, "force to migrate with deprecated cluster-tier-config")

	return cmd
}

// migrateImageCmd represents the migrate command
var migrateImageCmd = &cobra.Command{
	Use:   "image [gitops path]",
	Short: "migrate default images from gitops",
	Long:  `migrate default images from gitops`,
	Run: func(cmd *cobra.Command, args []string) {
		var opts []grpc.DialOption
		opts = append(opts, grpc.WithInsecure())
		conn, err := grpc.Dial(configCenterEndpoint, opts...)
		if err != nil {
			panic(err)
		}
		defer conn.Close()
		client := pb.NewConfigCenterServiceClient(conn)
		gitopsfile.ImportImagesFromGitops(client, args[0], "v6.1.0", "v99.99.99", true)
	},
}

func newMigrateTemplateCmd() *cobra.Command {
	var gitopsPath, tierPath, minVersion, maxVersion string
	var force bool
	cmd := &cobra.Command{
		Use:   "template",
		Short: "migrate templates from gitops",
		Long:  `migrate templates from gitops`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				panic(err)
			}
			defer conn.Close()
			client := pb.NewConfigCenterServiceClient(conn)

			if !force {
				return errors.New("node instance list is only available in SKU api, please use ops sync instead")
			}
			return gitopsfile.ImportDefaultTemplatesFromGitops(client, gitopsPath, minVersion, maxVersion, gitopsfile.GetAllInstancesFromTierConfig(tierPath))
		},
	}

	cmd.Flags().StringVar(&gitopsPath, "gitops", "/gitops-data/tidb-configuration", "local path of tidb-configuration repository")
	cmd.Flags().StringVar(&tierPath, "tier", "/gitops-data/cluster-tier-config", "local path of cluster-tier-config repository")
	cmd.Flags().StringVar(&minVersion, "from", "v6.1.0", "minimum version to migrate")
	cmd.Flags().StringVar(&maxVersion, "to", "v99.99.99", "max version to migrate")
	cmd.Flags().BoolVar(&force, "force", false, "force to migrate with deprecated cluster-tier-config")

	return cmd
}

func newMigrateInitSqlCmd() *cobra.Command {
	var gitopsPath, minVersion, maxVersion string
	cmd := &cobra.Command{
		Use:   "initsql",
		Short: "migrate initsql",
		Long:  `migrate initsql`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				panic(err)
			}
			defer conn.Close()
			client := pb.NewConfigCenterServiceClient(conn)
			return gitopsfile.ImportInitSql(client, gitopsPath, minVersion, maxVersion)
		},
	}

	cmd.Flags().StringVar(&gitopsPath, "gitops", "/gitops-data/tidb-configuration", "local path of tidb-configuration repository")
	cmd.Flags().StringVar(&minVersion, "from", "v6.1.0", "minimum version to migrate")
	cmd.Flags().StringVar(&maxVersion, "to", "v99.99.99", "max version to migrate")

	return cmd
}

func newMigrateTiproxyCmd() *cobra.Command {
	var version, tag string
	var setDefault bool
	cmd := &cobra.Command{
		Use:   "tiproxy",
		Short: "migrate tiproxy image, default version and config template to config-center",
		Long:  `migrate tiproxy image, default version and config template to config-center`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				return err
			}
			defer conn.Close()
			client := pb.NewConfigCenterServiceClient(conn)

			if tag == "" {
				tag = version
			}
			return gitopsfile.AddTiproxyVersion(client, version, tag, setDefault)
		},
	}

	cmd.Flags().StringVar(&version, "version", "v1.3.2", "tiproxy version")
	cmd.Flags().StringVar(&tag, "tag", "", "tiproxy image tag")
	cmd.Flags().BoolVar(&setDefault, "default", false, "set as default version")

	return cmd
}

func newMigrateClusterCmd() *cobra.Command {
	var gitopsPath, coreSvcEndpoint string
	var dryRun, all, fromFile bool
	cmd := &cobra.Command{
		Use:   "cluster [cluster-id]",
		Short: "migrate cluster config from gitops",
		Long:  `migrate cluster config from gitops`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				panic(err)
			}
			defer conn.Close()
			ccc := pb.NewConfigCenterServiceClient(conn)

			conn2, err := grpc.Dial(coreSvcEndpoint, opts...)
			if err != nil {
				panic(err)
			}
			coreC := core.NewCoreServiceClient(conn2)

			if fromFile {
				return importClusterConfigs(ccc, coreC, gitopsPath, args[0], dryRun)
			}

			kubeconfig := os.Getenv("KUBECONFIG")
			cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
			if err != nil {
				panic(err)
			}
			cloudv1.AddToScheme(scheme.Scheme)
			infraCli, err := client.New(cfg, client.Options{
				Scheme: scheme.Scheme,
			})

			if all {
				return tryToMigrateAllCluster2(ccc, coreC, infraCli)
			}

			clusterIDint, err := strconv.ParseUint(args[0], 10, 64)
			if err != nil {
				return err
			}
			resp, err := coreC.ListClusters(context.Background(), &core.ListClustersReq{
				Filter: &core.ClusterFilter{
					ClusterIds: []uint64{clusterIDint},
				},
			})
			if err != nil {
				return err
			}
			coreCluster := resp.GetClusters()[clusterIDint]
			if coreCluster == nil {
				return fmt.Errorf("cluster is not found in core-svc resp")
			}
			return tryImportClusterConfigWithValidate(ccc, infraCli, args[0], coreCluster.GetProjectId())
		},
	}

	cmd.Flags().StringVar(&gitopsPath, "gitops", "/gitops-data/tidb-configuration", "local path of tidb-configuration repository")
	cmd.Flags().StringVar(&coreSvcEndpoint, "core-svc", "", "core-svc endpoint")
	cmd.Flags().BoolVar(&dryRun, "dry-run", false, "not import, just show result")
	cmd.Flags().BoolVar(&all, "all", false, "try to migrate all alive cluster, force to use dry-run mode")
	cmd.Flags().BoolVar(&fromFile, "from-file", false, "migrate from local file instead of api-server")

	return cmd
}

func newMigrateTenantCmd() *cobra.Command {
	var gitopsPath string
	var dryRun, all bool
	cmd := &cobra.Command{
		Use:   "tenant [tenant-id]",
		Short: "migrate tenant config from gitops",
		Long:  `migrate tenant config from gitops`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				panic(err)
			}
			defer conn.Close()
			client := pb.NewConfigCenterServiceClient(conn)

			if all {
				return importAllTenantConfig(client, gitopsPath, dryRun)
			}
			tenantID, err := strconv.ParseUint(args[0], 10, 64)
			if err != nil {
				return err
			}
			return importTenantConfigFromGitops(client, gitopsPath, tenantID, dryRun)
		},
	}

	cmd.Flags().StringVar(&gitopsPath, "gitops", "/gitops-data/tidb-configuration", "local path of tidb-configuration repository")
	cmd.Flags().BoolVar(&dryRun, "dry-run", false, "not import, just show result")
	cmd.Flags().BoolVar(&all, "all", false, "import all tenants")

	return cmd
}

func newMigrateLightningCmd() *cobra.Command {
	var gitopsPath string
	var dryRun, all bool
	cmd := &cobra.Command{
		Use:   "lightning [tenant-id]",
		Short: "migrate lightning config from gitops",
		Long:  `migrate lightning config from gitops`,
		RunE: func(cmd *cobra.Command, args []string) error {
			var opts []grpc.DialOption
			opts = append(opts, grpc.WithInsecure())
			conn, err := grpc.Dial(configCenterEndpoint, opts...)
			if err != nil {
				panic(err)
			}
			defer conn.Close()
			client := pb.NewConfigCenterServiceClient(conn)

			if all {
				return importAllLightningConfig(client, gitopsPath, dryRun)
			}
			tenantID, err := strconv.ParseUint(args[0], 10, 64)
			if err != nil {
				return err
			}
			_, err = imoprtLightningOfOneTenant(client, gitopsPath, tenantID, dryRun)
			return err
		},
	}

	cmd.Flags().StringVar(&gitopsPath, "gitops", "/gitops-data/tidb-configuration", "local path of tidb-configuration repository")
	cmd.Flags().BoolVar(&dryRun, "dry-run", false, "not import, just show result")
	cmd.Flags().BoolVar(&all, "all", false, "import all tenants")

	return cmd
}

func init() {
	rootCmd.AddCommand(migrateCmd)

	migrateCmd.AddCommand(migrateImageCmd)
	migrateCmd.AddCommand(newMigrateTemplateCmd())
	migrateCmd.AddCommand(newMigrateInitSqlCmd())
	migrateCmd.AddCommand(newMigrateBaseCmd())
	migrateCmd.AddCommand(newMigrateTiproxyCmd())
	migrateCmd.AddCommand(newMigrateClusterCmd())
	migrateCmd.AddCommand(newMigrateTenantCmd())
	migrateCmd.AddCommand(newMigrateLightningCmd())
	migrateCmd.AddCommand(newMigrateNGBaseCmd())
	migrateCmd.AddCommand(newMigrateNGCDCCmd())
	migrateCmd.AddCommand(newMigrateNGBRCmd())
	migrateCmd.AddCommand(newUpdateImageCmd())
	migrateCmd.AddCommand(newMigrateByocInitSqlCmd())

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// testCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// testCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

type meta struct {
	Version    string               `yaml:"version"`
	Components map[string]component `yaml:"images"`
}

type component struct {
	BaseImage string `yaml:"baseImage"`
	Tag       string `yaml:"tag"`
}

func importClusterConfigs(ccc pb.ConfigCenterServiceClient, csc core.CoreServiceClient, gitopsPath, clusterID string, dryRun bool) error {
	clusterIDint, err := strconv.ParseUint(clusterID, 10, 64)
	if err != nil {
		return err
	}
	resp, err := csc.ListClusters(context.Background(), &core.ListClustersReq{
		Filter: &core.ClusterFilter{
			ClusterIds: []uint64{clusterIDint},
		},
	})
	coreCluster := resp.GetClusters()[clusterIDint]
	if coreCluster == nil {
		return fmt.Errorf("cluster is not found in core-svc resp")
	}

	cloudProvider := coreCluster.GetCloudProviderName()
	tenantID := coreCluster.GetTenantId()
	version := coreCluster.GetVersion()
	instanceMap := make(map[string]string)
	for _, c := range coreCluster.GetComponents() {
		instanceMap[strings.ToLower(strings.TrimPrefix(c.GetComponentType().String(), "COMPONENT_TYPE_"))] = c.GetComputeInstanceType()
	}
	instanceMap["backup"] = ""
	instanceMap["restore"] = ""

	resp2, err := importClusterConfigFromGitops(ccc, gitopsPath, tenantID, clusterID, version, cloudProvider, instanceMap, dryRun)
	if err != nil {
		return err
	}
	respText, err := json.MarshalIndent(resp2, "", "    ")
	fmt.Println(string(respText))

	return nil
}

func tryToMigrateAllCluster2(ccc pb.ConfigCenterServiceClient, csc core.CoreServiceClient, kubec client.Client) error {
	resp, err := csc.ListClusters(context.Background(), &core.ListClustersReq{
		Filter: &core.ClusterFilter{
			Deleted:   ptr.To(false),
			IsDevTier: ptr.To(false),
			//ExcludeStatuses: []core.ClusterStatus{core.ClusterStatus_CLUSTER_STATUS_INITIALIZING, core.ClusterStatus_CLUSTER_STATUS_PREPARING, core.ClusterStatus_CLUSTER_STATUS_PAUSED, core.ClusterStatus_CLUSTER_STATUS_SCALING},
		},
		Pager: &base.Pager{
			Page:     1,
			PageSize: 200,
		},
		WithDataPlaneInfo: true,
	})
	if err != nil {
		return err
	}
	count := len(resp.GetClusters())
	success := 0
	failed := 0
	different := 0
	for clusterIDint, coreCluster := range resp.GetClusters() {
		if coreCluster.GetAttributes().GetUseNewDedicatedArch() {
			count--
			continue
		}
		if coreCluster.GetDataPlaneInfo().ShootName != "" {
			count--
			continue
		}
		fmt.Println(clusterIDint)
		clusterID := strconv.FormatUint(clusterIDint, 10)

		infraCluster, err := gitopsmigrate.GetInfraCluster(context.Background(), kubec, clusterID, coreCluster.ProjectId)
		if err != nil {
			failed++
			fmt.Printf("get cluster from infra failed: %s\n", err)
			continue
		}
		meta, err := gitopsmigrate.ImportClusterConfigFromInfra(ccc, kubec, clusterID, infraCluster, true)
		if err != nil {
			failed++
			fmt.Printf("write config-center failed: %s\n", err)
			continue
		}

		configRevision, err := gitopsmigrate.GetConfigRevisionByCluster(context.Background(), kubec, infraCluster)
		if err != nil {
			failed++
			fmt.Printf("get configRevision failed: %s\n", err)
			continue
		}
		diff, err := gitopsmigrate.Diff(meta, configRevision, infraCluster.Spec.Cluster.TiFlash.InstanceType != "")
		if diff {
			different++
			fmt.Println(err)
		} else {
			success++
		}
		fmt.Printf("total: %d, succuss: %d, failed: %d, different: %d\n", count, success, failed, different)
	}
	return nil
}

func tryImportClusterConfigWithValidate(ccc pb.ConfigCenterServiceClient, kubec client.Client, clusterID string, projectID uint64) error {
	cluster, err := gitopsmigrate.GetInfraCluster(context.Background(), kubec, clusterID, projectID)
	if err != nil {
		return fmt.Errorf("get cluster from infra failed: %w", err)
	}

	meta, err := gitopsmigrate.ImportClusterConfigFromInfra(ccc, kubec, clusterID, cluster, true)
	if err != nil {
		return fmt.Errorf("write config-center failed: %w", err)
	}

	configRevision, err := gitopsmigrate.GetConfigRevisionByCluster(context.Background(), kubec, cluster)
	if err != nil {
		return fmt.Errorf("get configRevision failed: %w", err)
	}
	diff, err := gitopsmigrate.Diff(meta, configRevision, cluster.Spec.Cluster.TiFlash.InstanceType != "")
	if diff {
		return err
	}

	return nil
}

func diff(meta *pb.UpdateClusterMetaResponse, configRevision *cloudv1.ConfigRevision) bool {
	tidb := configRevision.Spec.TiDB
	if tidb != nil {
		config := getConfigFromSlice(meta.ModifiedConfigurationFiles, "tidb")
		text, err := tidb.Config.MarshalTOML()
		if err != nil {
			panic(err)
		}
		if string(text) != config.GetText() {
			fmt.Printf("--gitops--\n%s--end--\n", string(text))
			fmt.Printf("--config-center--\n%s--end--\n", config.GetText())
			return true
		}

		comp := getComponentFromSlice(meta.ModifiedComponents, "tidb")
		if *tidb.BaseImage != comp.GetBaseImage() || *tidb.Tag != comp.GetEffectiveImageTag() {
			fmt.Printf("--gitops--\n%s %s\n--end--\n", *tidb.BaseImage, *tidb.Tag)
			fmt.Printf("--config-center--\n%s %s\n--end--\n", comp.GetBaseImage(), comp.GetEffectiveImageTag())
			return true
		}
	}
	pd := configRevision.Spec.PD
	if pd != nil {
		config := getConfigFromSlice(meta.ModifiedConfigurationFiles, "pd")
		text, err := pd.Config.MarshalTOML()
		if err != nil {
			panic(err)
		}
		if string(text) != config.GetText() {
			fmt.Printf("--gitops--\n%s--end--\n", string(text))
			fmt.Printf("--config-center--\n%s--end--\n", config.GetText())
			return true
		}

		comp := getComponentFromSlice(meta.ModifiedComponents, "tidb")
		if *pd.BaseImage != comp.GetBaseImage() || *pd.Tag != comp.GetEffectiveImageTag() {
			fmt.Printf("--gitops--\n%s %s\n--end--\n", *pd.BaseImage, *pd.Tag)
			fmt.Printf("--config-center--\n%s %s\n--end--\n", comp.GetBaseImage(), comp.GetEffectiveImageTag())
			return true
		}
	}
	tikv := configRevision.Spec.TiKV
	if tikv != nil {
		config := getConfigFromSlice(meta.ModifiedConfigurationFiles, "tikv")
		text, err := tikv.Config.MarshalTOML()
		if err != nil {
			panic(err)
		}
		if string(text) != config.GetText() {
			fmt.Printf("--gitops--\n%s--end--\n", string(text))
			fmt.Printf("--config-center--\n%s--end--\n", config.GetText())
			return true
		}

		comp := getComponentFromSlice(meta.ModifiedComponents, "tikv")
		if *tikv.BaseImage != comp.GetBaseImage() || *tikv.Tag != comp.GetEffectiveImageTag() {
			fmt.Printf("--gitops--\n%s %s\n--end--\n", *tikv.BaseImage, *tikv.Tag)
			fmt.Printf("--config-center--\n%s %s\n--end--\n", comp.GetBaseImage(), comp.GetEffectiveImageTag())
			return true
		}
	}
	tiflash := configRevision.Spec.TiFlash
	if tiflash != nil {
		config := getConfigFromSlice(meta.ModifiedConfigurationFiles, "tiflash")
		text, err := tiflash.Config.MarshalTOML()
		if err != nil {
			panic(err)
		}
		if string(text) != config.GetText() {
			fmt.Printf("--gitops--\n%s--end--\n", string(text))
			fmt.Printf("--config-center--\n%s--end--\n", config.GetText())
			return true
		}

		comp := getComponentFromSlice(meta.ModifiedComponents, "tiflash")
		if *tiflash.BaseImage != comp.GetBaseImage() || *tiflash.Tag != comp.GetEffectiveImageTag() {
			fmt.Printf("--gitops--\n%s %s\n--end--\n", *tiflash.BaseImage, *tiflash.Tag)
			fmt.Printf("--config-center--\n%s %s\n--end--\n", comp.GetBaseImage(), comp.GetEffectiveImageTag())
			return true
		}
	}
	tiflashLearner := configRevision.Spec.TiFlashLearner
	if tiflashLearner != nil {
		config := getConfigFromSlice(meta.ModifiedConfigurationFiles, "tiflash_learner")
		text, err := tiflashLearner.Config.MarshalTOML()
		if err != nil {
			panic(err)
		}
		if string(text) != config.GetText() {
			fmt.Printf("--gitops--\n%s--end--\n", string(text))
			fmt.Printf("--config-center--\n%s--end--\n", config.GetText())
			return true
		}
	}

	return false
}

func getComponentFromSlice(components []*pb.ClusterComponent, name string) *pb.ClusterComponent {
	for _, c := range components {
		if c.ComponentName == name {
			return c
		}
	}
	return nil
}

func getConfigFromSlice(configs []*pb.ConfigurationFile, name string) *pb.ConfigurationFile {
	for _, c := range configs {
		if c.ConfigurationType == name {
			return c
		}
	}
	return nil
}

// import (version, cloud provider, instance) from args
// import (overwrite_image_tag,custom_params) from gitops
// get templateID from config-center search
// will lost base_image
// final config text may be different from value in use
// --- at cluster level
func importClusterConfigFromGitops(c pb.ConfigCenterServiceClient, gitopsPath string, tenantID uint64, clusterID, version, cloudProvider string, instance map[string]string, dryRun bool) (*pb.UpdateClusterMetaResponse, error) {
	metaReq := pb.UpdateClusterMetaRequest{
		ClusterType:  pb.ClusterType_DEDICATED_TIDB,
		ClusterId:    clusterID,
		Version:      &version,
		ValidateOnly: dryRun,
	}

	// 1. import config
	filenames := []string{"tidb", "tikv", "pd", "tiflash", "tiflash_learner", "restore", "backup", "ticdc"}
	for _, filename := range filenames {
		belongTo := filename
		if belongTo == "tiflash_learner" {
			belongTo = "tiflash"
		}

		//ignore non-exist componnet configs
		if _, ok := instance[belongTo]; !ok {
			continue
		}

		var tenantConfig, clusterConfig map[string]any
		var format pb.Format
		bt, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), gitopsfile.ToGitopsFilename(filename)))
		if err != nil && !os.IsNotExist(err) {
			return nil, err
		}
		if err == nil {
			if gitopsfile.ToGitopsFormat(filename) == "toml" {
				_, err = toml.NewDecoder(bytes.NewReader(bt)).Decode(&tenantConfig)
				if err != nil {
					panic(err)
				}
			} else {
				err = yaml.NewDecoder(bytes.NewReader(bt)).Decode(&tenantConfig)
				if err != nil {
					panic(err)
				}
			}
		}

		bc, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "clusters", fmt.Sprintf("cluster-%s", clusterID), gitopsfile.ToGitopsFilename(filename)))
		if err != nil && !os.IsNotExist(err) {
			return nil, err
		}
		if err == nil {
			if gitopsfile.ToGitopsFormat(filename) == "toml" {
				_, err = toml.NewDecoder(bytes.NewReader(bc)).Decode(&clusterConfig)
				if err != nil {
					return nil, err
				}
			} else {
				err = yaml.NewDecoder(bytes.NewReader(bc)).Decode(&clusterConfig)
				if err != nil {
					panic(err)
				}
			}
		}

		if filename == "backup" || filename == "restore" {
			format = pb.Format_JSON
		} else {
			format = pb.Format_TOML
		}

		tenantAndlusterConfig := utils.FlattenMap(utils.MergeRawConfig(tenantConfig, clusterConfig))
		customConfig := make(map[string]*pb.ParamItem)
		for k, v := range tenantAndlusterConfig {
			anyV, err := pb.NewValue(v)
			if err != nil {
				return nil, err
			}
			customConfig[k] = &pb.ParamItem{
				Value:   anyV,
				Comment: "",
			}
		}

		resp, err := c.SearchDefaultConfigurationTemplate(context.Background(), &pb.SearchDefaultConfigurationTemplateRequest{
			ClusterType:       pb.ClusterType_DEDICATED_TIDB,
			ConfigurationType: filename,
			Version:           version,
			CloudProvider:     cloudProvider,
			NodeInstance:      instance[filename],
		})
		if err != nil {
			return nil, err
		}

		templateID := resp.GetTemplateId()

		metaReq.ConfigurationFileRequests = append(metaReq.ConfigurationFileRequests, &pb.UpdateClusterConfigurationFileRequest{
			ConfigurationFilename: filename,
			ConfigurationType:     &filename,
			BelongToComponent:     &belongTo,
			Format:                &format,
			TemplateId:            &templateID,
			CustomParams:          customConfig,
			AllowMissing:          true,
		})

	}

	// import component
	components := []string{"tidb", "tikv", "pd", "tiflash", "restore", "backup", "ticdc"}
	// 1. get custom tag
	var finalMeta, metaC meta
	bmt, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "meta.yaml"))
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	err = yaml.Unmarshal(bmt, &finalMeta)
	if err != nil {
		return nil, err
	}
	cmt, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "clusters", fmt.Sprintf("cluster-%s", clusterID), "meta.yaml"))
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	err = yaml.Unmarshal(cmt, &metaC)
	for name, c := range metaC.Components {
		finalMeta.Components[name] = c
	}

	// 2. to config-center request
	for _, c := range components {
		nodeInstance, ok := instance[c]
		if !ok {
			// ignore non-exist comonent
			continue
		}
		tag := ""
		if cc, ok := finalMeta.Components[c]; ok {
			tag = cc.Tag
		}
		metaReq.ComponentRequests = append(metaReq.ComponentRequests, &pb.UpdateClusterComponentRequest{
			ComponentName:     c,
			Version:           &version,
			CloudProvider:     &cloudProvider,
			NodeInstance:      &nodeInstance,
			OverwriteImageTag: &tag,
			AllowMissing:      true,
		})
	}

	return c.UpdateClusterMeta(context.Background(), &metaReq)
}

func importTenantConfigFromGitops(c pb.ConfigCenterServiceClient, gitopsPath string, tenantID uint64, dryRun bool) error {
	batchReq := pb.BatchUpdateTenantInitialCustomParamsRequest{}
	// 1. import config
	filenames := []string{"tidb", "tikv", "pd", "tiflash", "tiflash_learner", "restore", "backup", "ticdc", "tidb-lightning"}
	for _, filename := range filenames {
		belongTo := filename
		if belongTo == "tiflash_learner" {
			belongTo = "tiflash"
		}

		var tenantConfig map[string]any
		bt, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), gitopsfile.ToGitopsFilename(filename)))
		if err != nil && !os.IsNotExist(err) {
			return err
		}
		if err == nil {
			if gitopsfile.ToGitopsFormat(filename) == "toml" {
				_, err = toml.NewDecoder(bytes.NewReader(bt)).Decode(&tenantConfig)
				if err != nil {
					return err
				}
			} else {
				err = yaml.Unmarshal(bt, &tenantConfig)
				if err != nil {
					return err
				}
			}
		}

		flattenTenantConfig := utils.FlattenMap(tenantConfig)
		if len(flattenTenantConfig) == 0 {
			continue
		}
		customConfig := make(map[string]*pb.ParamItem)
		for k, v := range flattenTenantConfig {
			anyV, err := pb.NewValue(v)
			if err != nil {
				return err
			}
			customConfig[k] = &pb.ParamItem{
				Value:   anyV,
				Comment: "",
			}
		}

		batchReq.Requests = append(batchReq.Requests, &pb.UpdateTenantInitialCustomParamsRequest{
			ClusterType:       pb.ClusterType_DEDICATED_TIDB,
			ConfigurationType: filename,
			TenantId:          tenantID,
			InitialParams:     customConfig,
		})

	}

	var finalMeta meta
	bmt, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "meta.yaml"))
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	err = yaml.Unmarshal(bmt, &finalMeta)
	if err != nil {
		return err
	}

	if finalMeta.Version != "" && semver.Compare(finalMeta.Version, "v6.1.0") < 0 {
		return fmt.Errorf("cannot migrate tenant with default version < v6.1.0")
	}

	fmt.Printf("---%d---\n", tenantID)
	if !dryRun {
		if len(batchReq.Requests) > 0 {
			resp, err := c.BatchUpdateTenantInitialCustomParams(context.Background(), &batchReq)
			if err != nil {
				return err
			}
			respText, err := json.MarshalIndent(resp, "", "    ")
			fmt.Println(string(respText))
		}

		if finalMeta.Version != "" {
			resp2, err := c.UpdateDefaultVersion(context.Background(), &pb.UpdateDefaultVersionRequest{
				ClusterType: pb.ClusterType_DEDICATED_TIDB,
				TenantId:    &tenantID,
				Component:   "tidb",
				Version:     finalMeta.Version,
			})
			if err != nil {
				return err
			}
			fmt.Println(resp2.Version)
		}
	} else {
		reqText, err := json.MarshalIndent(batchReq, "", "    ")
		if err != nil {
			return err
		}
		fmt.Println(string(reqText))
		fmt.Println(finalMeta.Version)
	}

	return nil
}

func importAllTenantConfig(c pb.ConfigCenterServiceClient, gitopsPath string, dryRun bool) error {
	tenants, err := os.ReadDir(filepath.Join(gitopsPath, "tenants"))
	if err != nil {
		return err
	}
	count := 0
	for _, tenantDir := range tenants {
		if !tenantDir.IsDir() || !strings.HasPrefix(tenantDir.Name(), "tenant-") {
			continue
		}
		tenantID, err := strconv.ParseUint(tenantDir.Name()[7:], 10, 64)
		if err != nil {
			return err
		}
		err = importTenantConfigFromGitops(c, gitopsPath, tenantID, dryRun)
		if err != nil {
			fmt.Printf("failed to import tenant %d\n", tenantID)
			fmt.Println(err)
		}
		count++
	}

	fmt.Println(count)
	return nil
}

func imoprtLightningOfOneTenant(c pb.ConfigCenterServiceClient, gitopsPath string, tenantID uint64, dryRun bool) (int, error) {
	configType := pb.DEDICATED_TIDB_CONFIG_TIDB_LIGHTNING

	var tenantConfig map[string]any
	// 1. read tenant overlay
	bt, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "lightning.toml"))
	if err != nil && !os.IsNotExist(err) {
		return 0, err
	}
	if err == nil {
		_, err = toml.NewDecoder(bytes.NewReader(bt)).Decode(&tenantConfig)
		if err != nil {
			return 0, err
		}
	}

	// 2. convert to flatten format
	flattenTenantConfig := utils.FlattenMap(tenantConfig)
	if len(flattenTenantConfig) >= 0 {
		customConfig := make(map[string]*pb.ParamItem)
		for k, v := range flattenTenantConfig {
			anyV, err := pb.NewValue(v)
			if err != nil {
				return 0, err
			}
			customConfig[k] = &pb.ParamItem{
				Value:   anyV,
				Comment: "",
			}
		}
		if len(customConfig) > 0 {
			// 3. send request or just dry-run
			req := &pb.UpdateTenantInitialCustomParamsRequest{
				ClusterType:       pb.ClusterType_DEDICATED_TIDB,
				ConfigurationType: configType,
				TenantId:          tenantID,
				InitialParams:     customConfig,
			}
			reqText, err := json.MarshalIndent(req, "", "    ")
			if err != nil {
				return 0, err
			}
			fmt.Println("---tenant init: ", tenantID, "---")
			fmt.Println(string(reqText))
			if !dryRun {
				resp, err := c.UpdateTenantInitialCustomParams(context.Background(), req)
				if err != nil {
					return 0, err
				}
				respText, _ := json.MarshalIndent(resp, "", "    ")
				fmt.Println(string(respText))
			}
		}
	}

	// migrate cluster overlay
	clusters, err := os.ReadDir(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "clusters"))
	if os.IsNotExist(err) {
		return 0, nil
	} else if err != nil {
		return 0, err
	}

	count := 0
	for _, clusterDir := range clusters {
		if !clusterDir.IsDir() || !strings.HasPrefix(clusterDir.Name(), "cluster-") {
			continue
		}
		clusterID := clusterDir.Name()[8:]

		clusterVersionresp, err := c.GetClusterComponent(context.Background(), &pb.GetClusterComponentRequest{
			ClusterType:   pb.ClusterType_DEDICATED_TIDB,
			ClusterId:     clusterID,
			ComponentName: "tidb",
		})
		if err != nil {
			if status.Code(err) == codes.NotFound {
				fmt.Printf("cluster %s is not found in config-center\n", clusterID)
				continue
			} else {
				return 0, err
			}
		}
		if clusterVersionresp.GetCloudProvider() == "azure" {
			fmt.Printf("cluster %s is azure cluster, skip\n", clusterID)
			continue
		}
		clusterVersion := clusterVersionresp.GetVersion()

		var clusterConfig map[string]any

		bc, err := os.ReadFile(filepath.Join(gitopsPath, "tenants", fmt.Sprintf("tenant-%d", tenantID), "clusters", clusterDir.Name(), "lightning.toml"))
		if os.IsNotExist(err) {
			// ignore non-exist config
			continue
		} else if err != nil {
			return 0, err
		}
		if err == nil {
			_, err = toml.NewDecoder(bytes.NewReader(bc)).Decode(&clusterConfig)
			if err != nil {
				return 0, err
			}
		}

		tenantAndlusterConfig := utils.FlattenMap(utils.MergeRawConfig(tenantConfig, clusterConfig))
		customConfig := make(map[string]*pb.ParamItem)
		for k, v := range tenantAndlusterConfig {
			anyV, err := pb.NewValue(v)
			if err != nil {
				return 0, err
			}
			customConfig[k] = &pb.ParamItem{
				Value:   anyV,
				Comment: "",
			}
		}

		resp, err := c.SearchDefaultConfigurationTemplate(context.Background(), &pb.SearchDefaultConfigurationTemplateRequest{
			ClusterType:       pb.ClusterType_DEDICATED_TIDB,
			ConfigurationType: configType,
			Version:           clusterVersion,
			CloudProvider:     "",
			NodeInstance:      "",
		})
		if err != nil {
			return 0, err
		}

		templateID := resp.GetTemplateId()

		cfgReq := &pb.UpdateClusterConfigurationFileRequest{
			ClusterType:           pb.ClusterType_DEDICATED_TIDB,
			ClusterId:             clusterID,
			ConfigurationFilename: configType,
			ConfigurationType:     &configType,
			BelongToComponent:     &configType,
			Format:                ptr.To(pb.Format_TOML),
			TemplateId:            &templateID,
			CustomParams:          customConfig,
			AllowMissing:          true,
			ValidateOnly:          dryRun,
		}

		cmpReq := &pb.UpdateClusterComponentRequest{
			ClusterType:   pb.ClusterType_DEDICATED_TIDB,
			ClusterId:     clusterID,
			ComponentName: configType,
			Version:       &clusterVersion,
			AllowMissing:  true,
			ValidateOnly:  dryRun,
		}

		cmp, err := c.UpdateClusterComponent(context.Background(), cmpReq)
		if err != nil {
			return 0, err
		}
		cmpText, _ := json.MarshalIndent(cmp, "", "    ")
		fmt.Println(string(cmpText))
		cfg, err := c.UpdateClusterConfigurationFile(context.Background(), cfgReq)
		if err != nil {
			return 0, err
		}
		// hide full params
		cfg.FullParams = nil
		cfgText, _ := json.MarshalIndent(cfg, "", "    ")
		fmt.Println(string(cfgText))
		count++
		fmt.Printf("migrated cluster %s\n", clusterID)
	}
	return count, nil
}

func importAllLightningConfig(c pb.ConfigCenterServiceClient, gitopsPath string, dryRun bool) error {
	tenants, err := os.ReadDir(filepath.Join(gitopsPath, "tenants"))
	if err != nil {
		return err
	}
	count := 0
	failed := 0
	for _, tenantDir := range tenants {
		if !tenantDir.IsDir() || !strings.HasPrefix(tenantDir.Name(), "tenant-") {
			continue
		}
		tenantID, err := strconv.ParseUint(tenantDir.Name()[7:], 10, 64)
		if err != nil {
			return err
		}
		c, err := imoprtLightningOfOneTenant(c, gitopsPath, tenantID, dryRun)
		if err != nil {
			fmt.Printf("failed to import tenant %d %v\n", tenantID, err)
			failed++
		}
		fmt.Printf("migrated %d cluster for tenant %d\n", c, tenantID)
		count++
	}

	fmt.Printf("tenant total %d, failed %d\n", count, failed)
	return nil
}

func tomlConfig2Params(tomlConfig string) map[string]*pb.ParamItem {
	var configStruct map[string]any
	_, err := toml.NewDecoder(bytes.NewReader([]byte(tomlConfig))).Decode(&configStruct)
	if err != nil {
		panic(err)
	}

	flattenConfig := utils.FlattenMap(configStruct)
	customConfig := make(map[string]*pb.ParamItem)
	for k, v := range flattenConfig {
		anyV, err := pb.NewValue(v)
		if err != nil {
			panic(err)
		}
		customConfig[k] = &pb.ParamItem{
			Value:   anyV,
			Comment: "",
		}
	}
	return customConfig
}