package eks

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/crossplane-contrib/provider-aws/apis/autoscaling/v1beta1"
	ec2v1alpha1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/log"

	"github.com/tidbcloud/eks-provider/controllers/cluster/task"
	clusterutils "github.com/tidbcloud/eks-provider/controllers/cluster/task/utils"
	cloudv1 "github.com/tidbcloud/infra-api/api/v1"
	"github.com/tidbcloud/infra-provider/pkg/api"
	"github.com/tidbcloud/infra-provider/pkg/cloud/aws"
	"github.com/tidbcloud/infra-provider/pkg/consts"
	"github.com/tidbcloud/infra-provider/pkg/deploy"
	"github.com/tidbcloud/infra-provider/pkg/k8s"
	infrak8s "github.com/tidbcloud/infra-provider/pkg/k8s"
	"github.com/tidbcloud/infra-provider/pkg/namer"
	"github.com/tidbcloud/infra-provider/pkg/nodegroup"
	"github.com/tidbcloud/infra-provider/pkg/nodegroup/selfmanaged"
	"github.com/tidbcloud/infra-provider/pkg/util"
	crossplaneutil "github.com/tidbcloud/infra-provider/pkg/util/crossplane"
)

// SelfManagedNodegroup is used to create/update/delete self-managed nodegroups for tidb cluster.
type SelfManagedNodegroup struct {
	*task.BaseTask
	awsCli aws.Client
}

func NewSelfManagedNodegroup(ctx *task.Context, awsCli aws.Client) *SelfManagedNodegroup {
	return &SelfManagedNodegroup{BaseTask: &task.BaseTask{Context: ctx}, awsCli: awsCli}
}

func (t *SelfManagedNodegroup) Name() string {
	return "SelfManagedNodegroup"
}

func (t *SelfManagedNodegroup) Create(ctx context.Context) deploy.Result {
	// skip it if self managed nodegroup is not enabled
	if util.GetAnnotation(t.Cluster, consts.UseSelfManagedNodegroupAnnotation) != consts.AnnoValTrue {
		return deploy.Done()
	}

	t.BeginAt = util.Pointer(time.Now())
	logger := log.FromContext(ctx)

	if t.TCGetter == nil || t.TCGetter.IsNil() {
		// Fatal to skip PostCreate
		return deploy.Fatal(fmt.Errorf("tidb cluster is not exist yet"))
	}

	// get the node system configuration from config
	nodeUserDataSystemConfiguration, err := t.Config.ClusterController.AWS.GetAWSUserdataConfigurationFromFile()
	if err != nil {
		return deploy.Error(fmt.Errorf("failed to get the userdata system configuration, %v", err))
	}

	cli := t.EnvProvider.GetClient()
	res, err := t.nodegroupSet(ctx, cli)
	if err != nil {
		return deploy.ErrorOrDone(err)
	}

	// Note res.NodeGroups only contains the nodegroup with desiredSize greater than 0 now.
	// There may be some other nodegroups we created before.
	expectedNames := make(map[string]struct{})
	for _, ng := range res.NodeGroups {
		// get all nodegroups for one component type in one zone of this cluster as `ng.Labels` indcludes zones label
		xpNgList := &v1beta1.AutoScalingGroupList{}
		if err := t.Local.List(ctx, xpNgList, client.MatchingLabels(ng.Labels)); err != nil {
			return deploy.ErrorOrDone(err)
		}

		// adjust the nodegroup name (ng.Name) for node scale up/down if needed
		newNg := t.adjustNgNameForInstanceType(ctx, xpNgList, &ng)
		if (newNg && nonDeletingASGCount(xpNgList) > 0) || nonDeletingASGCount(xpNgList) > 1 {
			logger.Info("detected scaling scenario",
				"component", ng.ComponentType,
				"newNodegroup", newNg,
				"existingNodegroupCount", nonDeletingASGCount(xpNgList),
				"instanceType", ng.MachineSpec.MachineType,
				"zone", ng.Zones)

			scalingOut := false
			comp := t.Cluster.GetComponent(ng.ComponentType)
			compStatus := t.Cluster.GetComponentStatus(ng.ComponentType)

			// if there are more than one nodegroups in one zone, and the component is scaling out and scaling down,
			// we need to scale out first, i.e. modify the nodegroup with previous instance type to scale out first.
			scalingDown, err := nodegroup.IsComponentScalingDown(t.Provider, comp, compStatus)
			if err != nil {
				return deploy.Error(fmt.Errorf("failed to check if the component %s is scaling down: %w", ng.ComponentType, err))
			}

			isScalingOut := nodegroup.IsComponentScalingOut(comp, compStatus)
			logger.Info("component scaling status check",
				"component", ng.ComponentType,
				"isScalingOut", isScalingOut,
				"isScalingDown", scalingDown,
				"currentUpdatedReplicas", compStatus.GetUpdatedReplicas(),
				"desiredReplicas", comp.GetReplicas())
			if nodegroup.IsComponentScalingOut(comp, compStatus) && scalingDown {
				logger.Info("handling simultaneous scale out and scale down scenario",
					"component", ng.ComponentType,
					"targetInstanceType", ng.MachineSpec.MachineType,
					"nodeGroupsInZone", len(xpNgList.Items))

				for _, xpNG := range xpNgList.Items {
					// NOTE(liubo02): for operator v2, it's no need to scale out the asg with previous instance type
					// Because the DesiredCapacity is unset in operator v2, we just change nothing
					currentInstanceType := selfmanaged.GetInstanceTypeFromStatus(&xpNG)
					if currentInstanceType != ng.MachineSpec.MachineType && xpNG.Spec.ForProvider.DesiredCapacity != nil && *xpNG.Spec.ForProvider.DesiredCapacity != int64(ng.Minimum) {
						logger.Info("scale out the nodegroup with the previous instance type",
							"name", xpNG.Name,
							"component", ng.ComponentType,
							"currentInstanceType", currentInstanceType,
							"targetInstanceType", ng.MachineSpec.MachineType,
							"currentDesiredCapacity", *xpNG.Spec.ForProvider.DesiredCapacity,
							"newDesiredCapacity", ng.Minimum,
							"newMinSize", ng.Minimum,
							"newMaxSize", ng.Maximum)

						xpNG.Spec.ForProvider.DesiredCapacity = ptr.To(int64(ng.Minimum))
						xpNG.Spec.ForProvider.MinSize = ptr.To(int64(ng.Minimum))
						xpNG.Spec.ForProvider.MaxSize = ptr.To(int64(ng.Maximum))
						if err := k8s.Apply(ctx, t.Local, &xpNG); err != nil {
							return deploy.Error(fmt.Errorf("failed to update nodegroup %s: %w", xpNG.Name, err))
						}
						scalingOut = true
						logger.Info("successfully updated nodegroup capacity for scale out",
							"name", xpNG.Name,
							"component", ng.ComponentType)
						break
					}
				}
			}

			// a new nodegroup should be created or is creating for instance type scale up/down
			// If it's scaling out, will not change the phase to `CreatingNodes`.
			currentPhase := api.GetInstanceTypeScalePhase(t.Cluster, ng.ComponentType)
			if !scalingOut && currentPhase == cloudv1.InstanceTypeScalePhaseNone {
				logger.Info("transitioning instance type scale phase to CreatingNodes",
					"component", ng.ComponentType,
					"currentPhase", currentPhase,
					"newPhase", cloudv1.InstanceTypeScalePhaseCreatingNodes,
					"targetInstanceType", ng.MachineSpec.MachineType,
					"reason", "new nodegroup creation needed")
				clusterutils.TransInstanceTypeScalePhase(t.Logger, t.Cluster, ng.ComponentType, cloudv1.InstanceTypeScalePhaseCreatingNodes)
			} else if scalingOut {
				logger.Info("skipping phase transition due to scaling out",
					"component", ng.ComponentType,
					"currentPhase", currentPhase,
					"scalingOut", scalingOut)
			}
		}

		sgIDs := make([]*string, 0, 2)
		// Add control plane SG's ID into launch template,
		// so that the node can communicate with the EKS control plane and managed nodegroups.
		eksSG := t.EnvProvider.AWS().GetEKSSecurityGroup()
		if eksSG == "" {
			return deploy.Error(fmt.Errorf("sg of eks %s is not found", t.EnvProvider.GetClusterName()))
		}
		sgIDs = append(sgIDs, &eksSG)

		// add dedicated SG's ID into launch template for clusters enabled dedicated SG
		if t.Cluster.FeatureEnabled(consts.ClusterFeatureDedicatedSG) {
			ec2cli := t.awsCli.EC2(t.GetAWSAssumedRoleForSDK())
			sg, err := retrieveServiceSG(ctx, ec2cli, t.Namer)
			if err != nil {
				return deploy.Error(err)
			}
			if sg == nil {
				return deploy.Error(fmt.Errorf("not found service security group for cluster"))
			}
			sgIDs = append(sgIDs, sg.GroupId)
		}

		group, ngLaunchTemplate, err := selfmanaged.GetTiDBClusterNodeGroup(
			t.Cluster,
			ng,
			t.GetNodeGroupName(ng.Name, ng.MachineSpec.MachineType, string(ng.ComponentType), ng.Zones, ""),
			t.EnvProvider.GetClusterName(),
			t.EnvProvider.GetClusterEndpoint(),
			t.EnvProvider.GetClusterCA(),
			t.EnvProvider.GetRegion(),
			t.GetNodeVersion(false),
			t.EnvProvider.AWS().GetNodeSubnets(ng.Zones...),
			sgIDs,
			t.GetAWSProviderConfigName(),
			t.GetAWSNodeRoleName(),
			t.Config.AMIConfig,
			nodeUserDataSystemConfiguration,
		) // NodeRole name as InstanceProfile name
		if err != nil {
			return deploy.Error(err)
		}

		if t.Cluster.FeatureEnabled(consts.ClusterFeatureUseTiDBOperatorV2) {
			// unset desired cap for all asg of operator v2
			group.Spec.ForProvider.DesiredCapacity = nil
			// set tag for autoscaler to scale from 0
			// see https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#how-can-i-scale-a-node-group-to-0
			// see also controllers/cluster/task/common/builder.buildNodeSelector
			// NOTE: node.kubernetes.io/instance-type and topology.kubernetes.io/zone will be automatically discovered
			group.Spec.ForProvider.Tags = append(group.Spec.ForProvider.Tags,
				&v1beta1.Tag{
					Key:               ptr.To("k8s.io/cluster-autoscaler/node-template/label/" + namer.ClusterSelectorKey),
					Value:             ptr.To(t.Namer.GetClusterName()),
					PropagateAtLaunch: ptr.To(true),
				},
				&v1beta1.Tag{
					Key:               ptr.To("k8s.io/cluster-autoscaler/node-template/label/" + namer.ComponentSelectorKey),
					Value:             ptr.To(string(ng.ComponentType)),
					PropagateAtLaunch: ptr.To(true),
				},
				&v1beta1.Tag{
					Key:               ptr.To("k8s.io/cluster-autoscaler/node-template/label/" + namer.NamespaceSelectorKey),
					Value:             ptr.To(t.Namer.GetTiDBNamespace()),
					PropagateAtLaunch: ptr.To(true),
				},
			)
		}

		logger.Info("create or update nodegroupLaunchTemplate", "name", group.Name, "minimum", ng.Minimum, "component", string(ng.ComponentType))
		if err := t.ignoreChanges(ctx, ngLaunchTemplate); err != nil {
			return deploy.Panic(err)
		}
		if err := k8s.Apply(ctx, t.Local, ngLaunchTemplate); err != nil {
			return deploy.Error(err)
		}

		logger.Info("create or update self-managed node group", "name", group.Name, "minimum", ng.Minimum, "component", string(ng.ComponentType))
		if err := t.ignoreChangesFromASG(ctx, group); err != nil {
			return deploy.Panic(err)
		}
		if err := k8s.Apply(ctx, t.Local, group); err != nil {
			return deploy.Error(err)
		}

		expectedNames[group.Name] = struct{}{}
	}

	// Set DesiredSize of nodegroup created before to 0.
	// (but do not touch any previous nodegroups when scaling for instance type, as they will be deleted after scale finished).
	// Consider delete it directly?
	groups, err := selfmanaged.GetTiDBClusterSelfManagedNodeGroups(ctx, t.Local, t.Namer.ClusterSelector(), t.Cluster)
	if err != nil {
		return deploy.Error(err)
	}
	for _, group := range groups {
		_, ok := expectedNames[group.Name]
		if ok {
			continue
		}

		if group.Spec.ForProvider.DesiredCapacity == nil || *group.Spec.ForProvider.DesiredCapacity > 0 {
			logger.Info("set desired size of nodegroup to 0", "nodegroup", group.Name, "component", group.Labels["tidb-component"])

			group.Spec.ForProvider.DesiredCapacity = ptr.To[int64](0)
			group.Spec.ForProvider.MaxSize = ptr.To[int64](1)
			group.Spec.ForProvider.MinSize = ptr.To[int64](0)
			err := k8s.Apply(ctx, t.Local, group)
			if err != nil {
				return deploy.Error(err)
			}
		}
	}

	return deploy.Done()
}

func (t *SelfManagedNodegroup) PostCreate(ctx context.Context) deploy.Result {
	// skip it if self managed nodegroup is not enabled
	if util.GetAnnotation(t.Cluster, consts.UseSelfManagedNodegroupAnnotation) != consts.AnnoValTrue {
		return deploy.Done()
	}

	logger := log.FromContext(ctx)
	cli := t.EnvProvider.GetClient()
	res, err := t.nodegroupSet(ctx, cli)
	if err != nil {
		return deploy.ErrorOrDone(err)
	}

	// group by component type
	componentNgs := make(map[cloudv1.ComponentType][]*cloudv1.NodegroupSpec)
	for i, ng := range res.NodeGroups {
		componentNgs[ng.ComponentType] = append(componentNgs[ng.ComponentType], &res.NodeGroups[i])
	}

	updateNeededComponents := make([]string, 0)
	errs := make([]error, 0) // errors for one component do not block the others.

outerForLoop:
	for componentType, ngs := range componentNgs {
		logger.Info("processing component nodegroups in PostCreate",
			"component", componentType,
			"nodeGroupCount", len(ngs))

		var moreThan1NgIn1Zone bool
		for _, ng := range ngs {
			// get all nodegroups for one component type in one zone of this cluster as `ng.Labels` indcludes zones label
			xpNgList := &v1beta1.AutoScalingGroupList{}
			if err = t.Local.List(ctx, xpNgList, client.MatchingLabels(ng.Labels)); err != nil {
				logger.Error(err, "failed to list nodegroups for component",
					"component", componentType,
					"zone", ng.Zones,
					"labels", ng.Labels)
				errs = append(errs, err)
				continue outerForLoop // handle next component
			}

			logger.Info("checking nodegroup active status",
				"component", componentType,
				"instanceType", ng.MachineSpec.MachineType,
				"zone", ng.Zones,
				"foundNodeGroups", len(xpNgList.Items))

			// check all nodegroups in all zones for one component become active
			if err = t.checkCurrentNgActive(ctx, cli, xpNgList, ng.MachineSpec.MachineType, componentType); err != nil {
				logger.Error(err, "nodegroup active check failed",
					"component", componentType,
					"instanceType", ng.MachineSpec.MachineType,
					"zone", ng.Zones)
				errs = append(errs, err)
				continue outerForLoop // handle next component
			}
			if len(xpNgList.Items) > 1 {
				moreThan1NgIn1Zone = true
			}
		}

		if !moreThan1NgIn1Zone {
			// no nodegroup update needed, no need to cordon nodes or delete nodegroups
			logger.Info("single nodegroup in zone, no scaling operations needed",
				"component", componentType)
			continue
		}

		itScalePhase := api.GetInstanceTypeScalePhase(t.Cluster, componentType)
		logger.Info("instance type scale phase check",
			"component", componentType,
			"currentPhase", itScalePhase,
			"multipleNodeGroupsInZone", moreThan1NgIn1Zone)

		if itScalePhase == cloudv1.InstanceTypeScalePhaseNone {
			logger.Info("scale phase is None, skipping scaling operations",
				"component", componentType,
				"reason", "deleting nodegroups has been called in the previous reconcile")
			continue // deleting nodegroups has been called in the previous reconcile
		}

		// get all nodegroups for one component type in all zonez of this cluster
		labels := make(map[string]string)
		for k, v := range ngs[0].Labels {
			// filter out all the labels related to the zones
			if k != consts.NodeGroupZonesLabelKey && k != consts.GardenerNodeAvailableZoneInfo && k != consts.GardenerWorkerNodeLabelName {
				labels[k] = v
			}
		}
		xpNgList := &v1beta1.AutoScalingGroupList{}
		if err = t.Local.List(ctx, xpNgList, client.MatchingLabels(labels)); err != nil {
			errs = append(errs, err)
			continue
		}

		if itScalePhase == cloudv1.InstanceTypeScalePhaseCreatingNodes || itScalePhase == cloudv1.InstanceTypeScalePhaseCordoningNodes {
			// cordon non-target nodes and uncordon target nodes
			logger.Info("entering node cordoning phase",
				"component", componentType,
				"currentPhase", itScalePhase,
				"targetInstanceType", ngs[0].MachineSpec.MachineType,
				"totalNodeGroups", len(xpNgList.Items))

			clusterutils.TransInstanceTypeScalePhase(t.Logger, t.Cluster, componentType, cloudv1.InstanceTypeScalePhaseCordoningNodes)
			if err = t.cordonUncordonNodes(ctx, cli, xpNgList, ngs[0].MachineSpec.MachineType, componentType); err != nil {
				logger.Error(err, "failed to cordon/uncordon nodes",
					"component", componentType,
					"targetInstanceType", ngs[0].MachineSpec.MachineType)
				errs = append(errs, err)
				continue
			}
			logger.Info("successfully completed node cordoning phase",
				"component", componentType,
				"targetInstanceType", ngs[0].MachineSpec.MachineType)
			itScalePhase = cloudv1.InstanceTypeScalePhaseCordoningNodes // already transferred to CordoningNodes
		}

		if itScalePhase == cloudv1.InstanceTypeScalePhaseCordoningNodes || itScalePhase == cloudv1.InstanceTypeScalePhaseRestartingPods {
			// rolling-update should be triggered after cordon/uncordon nodes
			// the TidbCluster component will be restarted in the next reconcile
			logger.Info("entering pod restarting phase",
				"component", componentType,
				"currentPhase", itScalePhase,
				"targetInstanceType", ngs[0].MachineSpec.MachineType)

			clusterutils.TransInstanceTypeScalePhase(t.Logger, t.Cluster, componentType, cloudv1.InstanceTypeScalePhaseRestartingPods)
			itScalePhase = cloudv1.InstanceTypeScalePhaseRestartingPods // already transferred to RestartingPods
			updateNeededComponents = append(updateNeededComponents, string(componentType))
			t.Logger.Info("TidbCluster is needed to rolling-update to target nodegroup",
				"component", componentType,
				"target-instance-type", ngs[0].MachineSpec.MachineType,
				"phase", "RestartingPods")
		}

		if itScalePhase == cloudv1.InstanceTypeScalePhaseRestartingPods || itScalePhase == cloudv1.InstanceTypeScalePhaseDeletingNodes {
			// delete previous nodegroups if updated TidbCluster to target nodegroups
			// will transfer to `InstanceTypeScalePhaseDeletingNodes` in this function if pods updated to the new nodegroup
			logger.Info("checking if ready to delete previous nodegroups",
				"component", componentType,
				"currentPhase", itScalePhase,
				"targetInstanceType", ngs[0].MachineSpec.MachineType,
				"nodeGroupsToCheck", len(xpNgList.Items))

			var deleted bool
			if deleted, err = t.waitAndDeletePreviousNgs(ctx, t.Cluster, cli, xpNgList, ngs[0].MachineSpec.MachineType, componentType); err != nil {
				logger.Error(err, "failed to wait and delete previous nodegroups",
					"component", componentType,
					"targetInstanceType", ngs[0].MachineSpec.MachineType)
				errs = append(errs, err)
				continue
			}
			if deleted {
				logger.Info("successfully deleted previous nodegroups",
					"component", componentType,
					"targetInstanceType", ngs[0].MachineSpec.MachineType)
				itScalePhase = cloudv1.InstanceTypeScalePhaseDeletingNodes // already transferred to DeletingNodes
			} else {
				logger.Info("waiting for conditions to delete previous nodegroups",
					"component", componentType,
					"targetInstanceType", ngs[0].MachineSpec.MachineType)
			}
		}

		if itScalePhase == cloudv1.InstanceTypeScalePhaseDeletingNodes {
			// once we called the deletion API for nodegroups, we mark the phase as None and let the deletion to be handled in the background
			logger.Info("completing instance type scaling process",
				"component", componentType,
				"currentPhase", itScalePhase,
				"newPhase", cloudv1.InstanceTypeScalePhaseNone,
				"reason", "deletion API called, letting background handle cleanup")
			clusterutils.TransInstanceTypeScalePhase(t.Logger, t.Cluster, componentType, cloudv1.InstanceTypeScalePhaseNone)
		}
	}

	if err = chooseNonRequeueOnlyErrorsFist(errs); err != nil {
		return deploy.Error(err)
	}

	if t.Cluster.Annotations == nil {
		t.Cluster.Annotations = make(map[string]string)
	}
	if len(updateNeededComponents) > 0 {
		// will update the TidbCluster in the next round of reconcile
		sort.Strings(updateNeededComponents)
		t.Cluster.Annotations[consts.ClusterUpdateAnnotationKeyInstanceType] = strings.Join(updateNeededComponents, consts.AnnoInstanceTypeSeparator)
	} else {
		// delete preivous annotation if no update needed so that it not affect the next reconcile triggered by another instance type scale
		delete(t.Cluster.Annotations, consts.ClusterUpdateAnnotationKeyInstanceType)
	}

	t.CommonK8sClusterStatus.NodeProvisioned = ptr.To(true)

	return deploy.Done()
}

func (t *SelfManagedNodegroup) Delete(ctx context.Context) deploy.Result {
	// skip it if self managed nodegroup is not enabled
	if util.GetAnnotation(t.Cluster, consts.UseSelfManagedNodegroupAnnotation) != consts.AnnoValTrue {
		return deploy.Done()
	}

	if err := t.Local.DeleteAllOf(ctx, &v1beta1.AutoScalingGroup{}, t.Namer.ClusterSelector()); err != nil {
		return deploy.Error(err)
	}

	// following patch is to fix issue 'cannot determine creation result - remove the crossplane.io/external-create-pending
	// annotation if it is safe to proceed' which will stuck the deletion
	ngs := &v1beta1.AutoScalingGroupList{}
	ops := k8s.ListOpts[*v1beta1.AutoScalingGroup]{
		ObjectList: ngs,
		Unpack:     func() []*v1beta1.AutoScalingGroup { return k8s.MapPtr(ngs.Items) },
	}
	if err := ops.PatchFilterBy(
		ctx,
		t.Local,
		func(_ *v1beta1.AutoScalingGroup) bool { return true },
		[]k8s.ObjectPatch{k8s.RemoveCrossplaneExternalCreatePending, k8s.DeletionTrigger},
		t.Namer.ClusterSelector(),
	); err != nil {
		return deploy.Error(err)
	}

	if err := t.Local.DeleteAllOf(ctx, &ec2v1alpha1.LaunchTemplate{}, t.Namer.ClusterSelector()); err != nil {
		return deploy.Error(err)
	}

	ngLaunchTemplates := &ec2v1alpha1.LaunchTemplateList{}
	ngLaunchTemplateOps := k8s.ListOpts[*ec2v1alpha1.LaunchTemplate]{
		ObjectList: ngLaunchTemplates,
		Unpack:     func() []*ec2v1alpha1.LaunchTemplate { return k8s.MapPtr(ngLaunchTemplates.Items) },
	}
	if err := ngLaunchTemplateOps.PatchFilterBy(
		ctx,
		t.Local,
		func(_ *ec2v1alpha1.LaunchTemplate) bool { return true },
		[]k8s.ObjectPatch{k8s.RemoveCrossplaneExternalCreatePending, k8s.DeletionTrigger},
		t.Namer.ClusterSelector(),
	); err != nil {
		return deploy.Error(err)
	}
	return deploy.Done()
}

func (t *SelfManagedNodegroup) PostDelete(ctx context.Context) deploy.Result {
	// skip it if self managed nodegroup is not enabled
	if util.GetAnnotation(t.Cluster, consts.UseSelfManagedNodegroupAnnotation) != consts.AnnoValTrue {
		return deploy.Done()
	}

	// TODO: Use another approach as only the node group names are required.

	ngs := &v1beta1.AutoScalingGroupList{}
	ops := k8s.ListOpts[*v1beta1.AutoScalingGroup]{
		ObjectList: ngs,
		Unpack:     func() []*v1beta1.AutoScalingGroup { return k8s.MapPtr(ngs.Items) },
	}
	list, err := ops.ListFilterBy(ctx, t.Local, func(_ *v1beta1.AutoScalingGroup) bool {
		return true
	}, t.Namer.ClusterSelector())
	if client.IgnoreNotFound(err) != nil {
		return deploy.Error(err)
	}

	if len(list) != 0 {
		var names []string
		for _, item := range list {
			names = append(names, item.Name)
		}
		return deploy.Error(fmt.Errorf("nodegroup not deleted, remain %d, names: %s", len(list), strings.Join(names, ",")))
	}

	ngLaunchTemplates := &ec2v1alpha1.LaunchTemplateList{}
	ngLaunchTemplateOps := k8s.ListOpts[*ec2v1alpha1.LaunchTemplate]{
		ObjectList: ngLaunchTemplates,
		Unpack:     func() []*ec2v1alpha1.LaunchTemplate { return k8s.MapPtr(ngLaunchTemplates.Items) },
	}
	ngLaunchTemplateList, err := ngLaunchTemplateOps.ListFilterBy(ctx, t.Local, func(_ *ec2v1alpha1.LaunchTemplate) bool {
		return true
	}, t.Namer.ClusterSelector())
	if client.IgnoreNotFound(err) != nil {
		return deploy.Error(err)
	}
	if len(ngLaunchTemplateList) != 0 {
		return deploy.Error(fmt.Errorf("nodegroup launchtemplate not deleted, remain %d", len(ngLaunchTemplateList)))
	}
	return deploy.Done()
}

// ignoreChanges gets the current launchtemplate CR, check if some fields are given,
// if so, use them instead of the new ones to ignore some fields changes to prevent recreating nodegroups.
func (t *SelfManagedNodegroup) ignoreChanges(ctx context.Context, newLaunchTemplate *ec2v1alpha1.LaunchTemplate) error {
	key := types.NamespacedName{
		Name: newLaunchTemplate.Name,
	}
	var launchTemplate ec2v1alpha1.LaunchTemplate
	if err := t.Local.Get(ctx, key, &launchTemplate); err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return err
	}

	// ignore AMI change
	// otherwise all tidb nodes will be recreated at the same time once we change the global config.
	if launchTemplate.Spec.ForProvider.LaunchTemplateData != nil && launchTemplate.Spec.ForProvider.LaunchTemplateData.ImageID != nil {
		newLaunchTemplate.Spec.ForProvider.LaunchTemplateData.ImageID = launchTemplate.Spec.ForProvider.LaunchTemplateData.ImageID
	}

	return nil
}

func (t *SelfManagedNodegroup) ignoreChangesFromASG(ctx context.Context, newASG *v1beta1.AutoScalingGroup) error {
	key := types.NamespacedName{
		Name: newASG.Name,
	}
	var asg v1beta1.AutoScalingGroup
	if err := t.Local.Get(ctx, key, &asg); err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return err
	}

	// ignore change the k8s version annotation.
	currentVersion := util.GetAnnotation(&asg, selfmanaged.K8sVersionAnnotationKey)
	if currentVersion != "" {
		newASG.Annotations[selfmanaged.K8sVersionAnnotationKey] = currentVersion
	}
	return nil
}

// setPrferPVCInfo sets topo.SchedulePreferPVC topo.CurrentPVCSpreads
// this magic make it run into the prefer pvc case when counting which zone we need to add node.
// ref https://github.com/tidbcloud/infra-provider/pull/1330/files
func (t *SelfManagedNodegroup) setPrferPVCInfo(ctx context.Context, topo *nodegroup.NodeTopology) error {
	topo.SchedulePreferPVC = true
	topo.CurrentPVCSpreads = make(map[cloudv1.ComponentType][]nodegroup.CountPerZone)

	tcClient := t.EnvProvider.GetClient()

	tcKey := types.NamespacedName{Namespace: t.TCGetter.Namespace(), Name: t.TCGetter.Name()}
	for _, comp := range topo.Components {
		component := comp.GetComponentType()
		pvcTopo, err := infrak8s.GetComponentPVCTopology(ctx, tcClient, tcKey, component)
		if err != nil {
			return fmt.Errorf("failed to get component pvc topology for %s: %w", component, err)
		}

		var zones []string
		for zone := range pvcTopo.Bound {
			zones = append(zones, zone)
		}
		sort.Strings(zones)

		var countPerZoneList []nodegroup.CountPerZone
		for _, zone := range zones {
			countPerZoneList = append(countPerZoneList, nodegroup.CountPerZone{
				Zone:  zone,
				Count: len(pvcTopo.Bound[zone]),
			})
		}

		topo.CurrentPVCSpreads[component] = countPerZoneList
	}

	log.FromContext(ctx).Info("set current pvc spreads", "spreds", topo.CurrentPVCSpreads)

	return nil
}

func (t *SelfManagedNodegroup) nodegroupSet(ctx context.Context, cli client.Client) (*nodegroup.Set, error) {
	topo, err := t.topologySpec(ctx, cli)
	if err != nil {
		return nil, err
	}

	err = t.setPrferPVCInfo(ctx, topo)
	if err != nil {
		return nil, err
	}

	builder := &nodegroup.Builder{ConfigProvider: t.ConfigProvider}

	if t.Cluster.FeatureEnabled(consts.ClusterFeatureUseTiDBOperatorV2) {
		builder.IgnoreReplicaValidation = true
		builder.ExtraTiDBNode = true
	}

	res, err := builder.Build(topo, false, false)
	if err != nil {
		return nil, err
	}
	for i := range res.NodeGroups {
		ng := &res.NodeGroups[i]
		ng.Name = tidbNodeGroupName(ng.Name)
		// used when deleting the resources
		ngNamer := t.Namer.WithComponent(string(ng.ComponentType))
		ng.Labels = util.ConcatenateTags(ng.Labels, ngNamer.Labels())
		// filter out the instance type label to avoid breaking scaling up label selector
		ng.Labels = getNodeLabelsForScaleUp(ng.Labels)
		appendZonesLabelForNodegroup(ng)
	}
	return res, nil
}

func (t *SelfManagedNodegroup) topologySpec(ctx context.Context, cli client.Client) (*nodegroup.NodeTopology, error) {
	spreads, err := t.currentSpreads(ctx, cli)
	if err != nil {
		return nil, err
	}
	zones := t.Zones()
	topo := &nodegroup.NodeTopology{
		Cluster:                        t.Cluster,
		Provider:                       t.Provider,
		Components:                     []cloudv1.ClusterComponent{},
		ComponentCurrentActualReplicas: make(map[cloudv1.ComponentType]int),
		Zones:                          zones,
		Tags:                           cloudv1.TagsFromLabels(t.Cluster.Labels),
		// Infra API builds the node group name from the owner name. So in order to avoid name collision
		// we need to use the cluster name here (which is unique when central creates the cluster).
		OwnerName:      t.Cluster.Name,
		CurrentSpreads: spreads,
	}

	for _, c := range []cloudv1.ClusterComponent{
		&t.Cluster.Spec.Cluster.PD,
		&t.Cluster.Spec.Cluster.TiKV,
		&t.Cluster.Spec.Cluster.TiDB,
		&t.Cluster.Spec.Cluster.TiProxy,
		&t.Cluster.Spec.Cluster.TiFlash,
	} {
		topo.Components = append(topo.Components, component{c, t.getDesiredReplicasCount(c)})
		topo.ComponentCurrentActualReplicas[c.GetComponentType()] = int(t.TCGetter.ComponentActualReplicas(c.GetComponentType()))
	}

	return topo, nil
}

func (t *SelfManagedNodegroup) currentSpreads(ctx context.Context, cli client.Client) (map[cloudv1.ComponentType][]nodegroup.CountPerZone, error) {
	spreads := make(map[cloudv1.ComponentType][]nodegroup.CountPerZone)
	for _, comp := range t.Cluster.GetComponents() {
		c, err := t.perZone(ctx, cli, comp.GetComponentType())
		if err != nil {
			return nil, err
		}
		if c != nil {
			spreads[comp.GetComponentType()] = c
		}
	}
	return spreads, nil
}

func (t *SelfManagedNodegroup) perZone(ctx context.Context, cli client.Client, ct cloudv1.ComponentType) ([]nodegroup.CountPerZone, error) {
	tcKey := types.NamespacedName{Namespace: t.TCGetter.Namespace(), Name: t.TCGetter.Name()}
	topo, err := infrak8s.GetComponentTopology(ctx, cli, tcKey, ct)
	if err != nil {
		return nil, err
	}
	var zones []string
	for zone := range topo.Scheduled {
		zones = append(zones, zone)
	}
	sort.Strings(zones)
	var dist []nodegroup.CountPerZone
	for _, zone := range zones {
		dist = append(dist, nodegroup.CountPerZone{
			Zone:  zone,
			Count: len(topo.Scheduled[zone]),
		})
	}
	return dist, nil
}

// adjust node group name for the current instance-type
// so that only the newly nodegroup needs to be created or matched nodegroup needs to be updated,
// and leave the previous nodegroups unchanged.
// return whether a new nodegroup should be created.
// NOTE: for new cluster without network, ng.Name is not used, so it only check the instance type of this asg.
func (t *SelfManagedNodegroup) adjustNgNameForInstanceType(ctx context.Context, xpNgList *v1beta1.AutoScalingGroupList, ng *cloudv1.NodegroupSpec) (newNg bool) {
	instanceTypeHash := util.FnvHash(ng.MachineSpec.MachineType)

	t.Logger.Info("adjusting nodegroup name for instance type",
		"component", ng.ComponentType,
		"targetInstanceType", ng.MachineSpec.MachineType,
		"instanceTypeHash", instanceTypeHash,
		"originalNodeGroupName", ng.Name,
		"existingNodeGroups", len(xpNgList.Items))

	// Log details of existing nodegroups for debugging
	for i, existingNg := range xpNgList.Items {
		existingInstanceType := getInstanceType(ctx, t.Local, &existingNg)
		t.Logger.Info("found existing nodegroup",
			"index", i,
			"nodeGroupName", existingNg.Name,
			"instanceType", existingInstanceType,
			"component", ng.ComponentType)
	}

	matchedNg := getSelfManagedNGByInstanceType(ctx, t.Local, xpNgList, ng.MachineSpec.MachineType)
	if matchedNg == nil {
		// no existing nodegroup's instance type matched (no nodegroup or mismatched), a new nodegroup should be created
		// always includes the instance-type hash part in the nodegroup name for the newly created nodegroup
		originalName := ng.Name
		ng.Name = fmt.Sprintf("%s-%s", ng.Name, instanceTypeHash)
		t.Logger.Info("No existing nodegroup match the instance type and a new nodegroup will be created",
			"component", ng.ComponentType,
			"instance-type", ng.MachineSpec.MachineType,
			"originalName", originalName,
			"newNodeGroupName", ng.Name,
			"instanceTypeHash", instanceTypeHash,
			"reason", "no matching instance type found")
		return true
	}

	// an existing nodegroup's instance type matched, it should be updated (the name should not be changed)
	// for nodegroup's name format, ref `nodegroup.FromNodegroupSpec`
	t.Logger.Info("found matching nodegroup for instance type",
		"component", ng.ComponentType,
		"matchedNodeGroup", matchedNg.Name,
		"instanceType", ng.MachineSpec.MachineType,
		"originalNodeGroupName", ng.Name)

	suffix := strings.TrimPrefix(matchedNg.Name, fmt.Sprintf("%s-", ng.Name))
	if strings.HasPrefix(suffix, instanceTypeHash) {
		// the current nodegroup name includes the instance-type hash part, keep the hash
		originalName := ng.Name
		ng.Name = fmt.Sprintf("%s-%s", ng.Name, instanceTypeHash)
		t.Logger.Info("An existing nodegroup matched the instance type, with instance-type hash in name",
			"component", ng.ComponentType,
			"instance-type", ng.MachineSpec.MachineType,
			"originalName", originalName,
			"finalNodeGroupName", ng.Name,
			"matchedNodeGroup", matchedNg.Name,
			"instanceTypeHash", instanceTypeHash)
	} else {
		t.Logger.Info("An existing nodegroup matched the instance type, without instance-type hash in name",
			"component", ng.ComponentType,
			"instance-type", ng.MachineSpec.MachineType,
			"finalNodeGroupName", ng.Name,
			"matchedNodeGroup", matchedNg.Name,
			"suffix", suffix,
			"instanceTypeHash", instanceTypeHash)
	}
	return false
}

func nonDeletingASGCount(xpNgList *v1beta1.AutoScalingGroupList) int {
	count := 0
	for _, ng := range xpNgList.Items {
		if ng.DeletionTimestamp.IsZero() {
			count++
		}
	}
	return count
}

func getInstanceType(ctx context.Context, cli client.Client, ng *v1beta1.AutoScalingGroup) *string {
	launchTemplateSpec := ng.Spec.ForProvider.LaunchTemplate

	if launchTemplateSpec != nil && launchTemplateSpec.LaunchTemplateName != nil {
		var launchTemplate ec2v1alpha1.LaunchTemplate
		if err := cli.Get(ctx, types.NamespacedName{Name: *launchTemplateSpec.LaunchTemplateName}, &launchTemplate); err == nil {
			if launchTemplate.Spec.ForProvider.LaunchTemplateData != nil {
				return launchTemplate.Spec.ForProvider.LaunchTemplateData.InstanceType
			}
		}
	}

	return nil
}

func getSelfManagedNGByInstanceType(ctx context.Context, cli client.Client, xpNgList *v1beta1.AutoScalingGroupList, instanceType string) *v1beta1.AutoScalingGroup {
	for i := range xpNgList.Items {
		if it := getInstanceType(ctx, cli, &xpNgList.Items[i]); it != nil && *it == instanceType {
			return &xpNgList.Items[i]
		}
	}
	return nil
}

func (t *SelfManagedNodegroup) checkCurrentNgActive(ctx context.Context, eksCli client.Client, xpNgList *v1beta1.AutoScalingGroupList, instanceType string, componentType cloudv1.ComponentType) error {
	// only check current instance type active status
	ng := getSelfManagedNGByInstanceType(ctx, t.Local, xpNgList, instanceType)
	if ng == nil {
		t.Logger.Error(nil, "nodegroup not found for instance type",
			"component", componentType,
			"instanceType", instanceType,
			"availableNodeGroups", len(xpNgList.Items))
		return fmt.Errorf("nodegroup for component %s with instance type %s not exist", componentType, instanceType)
	}

	t.Logger.Info("checking nodegroup readiness",
		"nodegroup", ng.Name,
		"component", componentType,
		"instanceType", instanceType,
		"conditions", len(ng.Status.Conditions))

	// No need to check synced. The spec may have been changed but xp has not handled.
	ready := crossplaneutil.IsReady(ng.Status.Conditions)
	if !ready {
		t.Logger.Info("nodegroup not ready",
			"nodegroup", ng.Name,
			"component", componentType,
			"instanceType", instanceType,
			"totalNodeGroups", len(xpNgList.Items),
			"conditions", ng.Status.Conditions)

		if len(xpNgList.Items) == 1 {
			// only one nodegroup exist, so the cluster should be Unavailable before the nodegroup become active
			return fmt.Errorf("non-active nodegroup %s for component %s with instance type %s", ng.Name, componentType, instanceType)
		}
		// node scaling from one nodegroup to another nodegroup (and waiting for the target nodegroup become ready),
		// so the cluster phase should not be changed
		t.Logger.Info("Waiting for the nodegroup become active before updating to target instance type",
			"nodegroup", ng.Name, "instance-type", instanceType, "component", componentType)
		return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
	}

	t.Logger.Info("nodegroup is ready, checking node join status",
		"nodegroup", ng.Name,
		"component", componentType,
		"instanceType", instanceType)

	// Check if all instances in the nodegroup join the EKS cluster
	var nodeList corev1.NodeList
	if err := eksCli.List(ctx, &nodeList, client.MatchingLabelsSelector{
		Selector: labels.SelectorFromSet(map[string]string{
			"Name": ng.Name,
		}),
	}); err != nil {
		t.Logger.Error(err, "failed to list nodes", "nodegroup", ng.Name)
		return err
	}
	healthyCount := numberOfHealthyInstances(ng)
	t.Logger.Info("checking node and instance counts",
		"nodegroup", ng.Name,
		"component", componentType,
		"instanceType", instanceType,
		"joinedNodes", len(nodeList.Items),
		"healthyInstances", healthyCount,
		"totalInstances", len(ng.Status.AtProvider.Instances))

	if len(nodeList.Items) != healthyCount {
		t.Logger.Info("waiting for all instances in the nodegroup join the EKS cluster",
			"nodegroup", ng.Name,
			"component", componentType,
			"current", len(nodeList.Items),
			"expect", healthyCount,
			"instanceType", instanceType)
		return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
	}

	if t.Cluster.FeatureEnabled(consts.ClusterFeatureUseTiDBOperatorV2) {
		// in operator v2, desired capacity is null, so use min size to check whether the number of instances is expected
		minSize := ng.Spec.ForProvider.MinSize
		maxSize := ng.Spec.ForProvider.MaxSize

		t.Logger.Info("validating capacity for TiDB Operator V2",
			"nodegroup", ng.Name,
			"component", componentType,
			"healthyInstances", healthyCount,
			"minSize", minSize,
			"maxSize", maxSize,
			"instanceType", instanceType)

		if minSize == nil || maxSize == nil {
			t.Logger.Error(nil, "min or max size is nil for operator v2",
				"nodegroup", ng.Name,
				"component", componentType,
				"minSize", minSize,
				"maxSize", maxSize)
			return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}

		if int64(healthyCount) < *minSize {
			t.Logger.Info("healthy instances below minimum size",
				"nodegroup", ng.Name,
				"component", componentType,
				"healthyInstances", healthyCount,
				"minSize", *minSize,
				"instanceType", instanceType)
			return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}

		if int64(healthyCount) > *maxSize {
			t.Logger.Info("healthy instances above maximum size",
				"nodegroup", ng.Name,
				"component", componentType,
				"healthyInstances", healthyCount,
				"maxSize", *maxSize,
				"instanceType", instanceType)
			return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}
	} else {
		// for operator v1, check desired capacity
		desiredCapacity := ng.Spec.ForProvider.DesiredCapacity

		t.Logger.Info("validating capacity for TiDB Operator V1",
			"nodegroup", ng.Name,
			"component", componentType,
			"healthyInstances", healthyCount,
			"desiredCapacity", desiredCapacity,
			"instanceType", instanceType)

		if desiredCapacity == nil {
			t.Logger.Error(nil, "desired capacity is nil for operator v1",
				"nodegroup", ng.Name,
				"component", componentType)
			return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}

		if int64(healthyCount) != *desiredCapacity {
			t.Logger.Info("healthy instances do not match desired capacity",
				"nodegroup", ng.Name,
				"component", componentType,
				"healthyInstances", healthyCount,
				"desiredCapacity", *desiredCapacity,
				"instanceType", instanceType)
			return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}
	}

	t.Logger.Info("nodegroup capacity validation passed",
		"nodegroup", ng.Name,
		"component", componentType,
		"instanceType", instanceType,
		"healthyInstances", healthyCount)

	return nil
}

// numberOfHealthyInstances returns the number of healthy instances in the ASG.
func numberOfHealthyInstances(asg *v1beta1.AutoScalingGroup) int {
	count := 0
	for _, instance := range asg.Status.AtProvider.Instances {
		if instance.LifecycleState != nil && *instance.LifecycleState == string(v1beta1.LifecycleState_InService) && *instance.HealthStatus == "Healthy" {
			count++
		}
	}
	return count
}

// cordonUncordonNodes cordons nodes in the old nodegroup and uncordons nodes in the new nodegroup.
func (t *SelfManagedNodegroup) cordonUncordonNodes(ctx context.Context, cli client.Client, xpNgList *v1beta1.AutoScalingGroupList, targetInstanceType string, componentType cloudv1.ComponentType) error {
outerForLoop:
	for _, xpNg := range xpNgList.Items {
		nodes := &corev1.NodeList{}
		ops := k8s.ListOpts[*corev1.Node]{
			ObjectList: nodes,
			Unpack:     func() []*corev1.Node { return k8s.MapPtr(nodes.Items) },
		}

		if it := getInstanceType(ctx, t.Local, &xpNg); it != nil && *it == targetInstanceType {
			// uncordon as target instance type may be changed back
			if err := ops.PatchFilterBy(ctx, cli, func(n *corev1.Node) bool { return true },
				[]k8s.ObjectPatch{k8s.UncordonNode}, client.MatchingLabels{"Name": xpNg.Name}); err != nil {
				t.Logger.Error(err, "Failed to uncordon nodes", "nodegroup", xpNg.Name)
				// failed to uncordon nodes should not change the cluster phase and only retry is enough
				return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
			}
			continue outerForLoop
		}

		// NOTE: append new taints for cross-plane NodeGroup will not apply to nodes, so we need to cordon them directly
		t.Logger.Info("Cordon nodes for nodegroup", "component", componentType, "nodegroup", xpNg.Name)
		if err := ops.PatchFilterBy(ctx, cli, func(n *corev1.Node) bool { return true },
			[]k8s.ObjectPatch{k8s.CordonNode}, client.MatchingLabels{"Name": xpNg.Name}); err != nil {
			t.Logger.Error(err, "Failed to cordon nodes", "nodegroup", xpNg.Name)
			// failed to cordon nodes should not change the cluster phase and only retry is enough
			return k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}
	}
	return nil
}

func (t *SelfManagedNodegroup) waitAndDeletePreviousNgs(ctx context.Context, cluster *cloudv1.Cluster, remoteCli client.Client, xpNgList *v1beta1.AutoScalingGroupList, targetInstanceType string, componentType cloudv1.ComponentType) (bool, error) {
	var (
		componentInstanceType = t.TCGetter.ComponentAnnotations(componentType)[consts.TcInstanceTypeAnnotationKey]
		componentReady        = t.TCGetter.ComponentAllInstancesReady(componentType)
	)

	t.Logger.Info("checking component readiness for nodegroup deletion",
		"component", componentType,
		"componentInstanceType", componentInstanceType,
		"targetInstanceType", targetInstanceType,
		"componentReady", componentReady,
		"nodeGroupsToCheck", len(xpNgList.Items))

	if componentInstanceType != targetInstanceType {
		// wait for the instance type update to be triggered
		// NOTE: a reconcile should be triggered after setting `TcInstanceTypeAnnotationKey`
		t.Logger.Info("Waiting for the instance type update to be triggered",
			"component", componentType,
			"current-instance-type", componentInstanceType,
			"target-instance-type", targetInstanceType,
			"reason", "component annotation not yet updated")
		return false, nil // no error, so that we can set `ClusterUpdateAnnotationKeyInstanceType`
	}

	if !componentReady {
		t.Logger.Info("Waiting for the component to be ready before deleting previous nodegroups",
			"component", componentType,
			"componentInstanceType", componentInstanceType,
			"targetInstanceType", targetInstanceType)
		// the cluster phase should not be changed if the pods is updating to the target instance type
		return false, k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
	}

	// check any pod has updated to the target instance type to avoid no-resync happens in tidb-operator (and `componentReady` is out of date)
	podLabels := map[string]string{
		consts.ComponentLabelKey: string(componentType),
		consts.InstanceLabelKey:  t.TCGetter.Name(),
	}

	t.Logger.Info("checking pod instance type annotations",
		"component", componentType,
		"namespace", t.TCGetter.Namespace(),
		"podLabels", podLabels,
		"targetInstanceType", targetInstanceType)

	pods, err := infrak8s.SelectPods(ctx, remoteCli, t.TCGetter.Namespace(), podLabels)
	if err != nil {
		t.Logger.Error(err, "Failed to select pods",
			"component", componentType,
			"namespace", t.TCGetter.Namespace(),
			"podLabels", podLabels)
		// the cluster phase should not be changed and retry is enough
		return false, k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
	}

	t.Logger.Info("found pods for component",
		"component", componentType,
		"podCount", len(pods),
		"targetInstanceType", targetInstanceType)

	var podsNotUpdated []string
	for _, pod := range pods {
		currentInstanceType, hasAnnotation := pod.Annotations[consts.TcInstanceTypeAnnotationKey]
		if !hasAnnotation || currentInstanceType != targetInstanceType {
			podsNotUpdated = append(podsNotUpdated, pod.Name)
			t.Logger.Info("pod not yet updated to target instance type",
				"component", componentType,
				"podName", pod.Name,
				"podNamespace", pod.Namespace,
				"currentInstanceType", currentInstanceType,
				"hasInstanceTypeAnnotation", hasAnnotation,
				"targetInstanceType", targetInstanceType,
				"podNodeName", pod.Spec.NodeName)
		}
	}

	if len(podsNotUpdated) > 0 {
		t.Logger.Info("Waiting for pods to be updated to the target instance type before deleting previous nodegroups",
			"component", componentType,
			"targetInstanceType", targetInstanceType,
			"totalPods", len(pods),
			"podsNotUpdated", len(podsNotUpdated),
			"podNames", podsNotUpdated)
		// the cluster phase should not be changed if the pods is updating to the target instance type
		return false, k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
	}

	t.Logger.Info("all pods have been updated to target instance type",
		"component", componentType,
		"targetInstanceType", targetInstanceType,
		"totalPods", len(pods))

	clusterutils.TransInstanceTypeScalePhase(t.Logger, cluster, componentType, cloudv1.InstanceTypeScalePhaseDeletingNodes)

outerForLoop:
	for _, xpNg := range xpNgList.Items {
		currentInstanceType := getInstanceType(ctx, t.Local, &xpNg)
		if currentInstanceType != nil && *currentInstanceType == targetInstanceType {
			t.Logger.Info("skipping deletion of target nodegroup",
				"component", componentType,
				"nodegroup", xpNg.Name,
				"instanceType", *currentInstanceType,
				"targetInstanceType", targetInstanceType)
			continue outerForLoop
		}

		// Log detailed information about the nodegroup being deleted
		healthyInstances := numberOfHealthyInstances(&xpNg)
		t.Logger.Info("preparing to delete previous nodegroup",
			"component", componentType,
			"nodegroup", xpNg.Name,
			"currentInstanceType", currentInstanceType,
			"targetInstanceType", targetInstanceType,
			"healthyInstances", healthyInstances,
			"totalInstances", len(xpNg.Status.AtProvider.Instances),
			"desiredCapacity", xpNg.Spec.ForProvider.DesiredCapacity,
			"minSize", xpNg.Spec.ForProvider.MinSize,
			"maxSize", xpNg.Spec.ForProvider.MaxSize)

		t.Logger.Info("Deleting nodegroup", "component", componentType, "nodegroup", xpNg.Name)
		if err := k8s.DeleteObject(ctx, t.Local, &xpNg); err != nil {
			t.Logger.Error(err, "Failed to delete nodegroup",
				"component", componentType,
				"nodegroup", xpNg.Name,
				"instanceType", currentInstanceType,
				"errorType", fmt.Sprintf("%T", err))
			// the cluster phase should not be changed and retry is enough
			return false, k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}

		t.Logger.Info("successfully initiated nodegroup deletion",
			"component", componentType,
			"nodegroup", xpNg.Name,
			"instanceType", currentInstanceType)

		t.Logger.Info("Deleting nodegroup launchtemplate", "component", componentType, "nodegroup", xpNg.Name)
		ngLaunchTemplate := &ec2v1alpha1.LaunchTemplate{
			ObjectMeta: metav1.ObjectMeta{
				Name: xpNg.Name,
			},
		}
		if err := k8s.DeleteObject(ctx, t.Local, ngLaunchTemplate); err != nil {
			t.Logger.Error(err, "Failed to delete launchtemplate",
				"component", componentType,
				"launchtemplate", xpNg.Name,
				"nodegroup", xpNg.Name,
				"errorType", fmt.Sprintf("%T", err))
			return false, k8s.NewRetriableError(k8s.RequeueOnlyError, defaultNodeGroupRequeue)
		}

		t.Logger.Info("successfully initiated launchtemplate deletion",
			"component", componentType,
			"launchtemplate", xpNg.Name,
			"nodegroup", xpNg.Name)
	}

	return true, nil
}

func (t *SelfManagedNodegroup) getDesiredReplicasCount(c cloudv1.ClusterComponent) int {
	desired := int(t.TCGetter.ComponentDesiredReplicas(c.GetComponentType()))
	if desired == 0 {
		return 0
	}
	if desired > c.GetReplicas() {
		return desired
	}
	return c.GetReplicas()
}