package zoneredundantevicter

import (
	"context"
	"encoding/json"
	"testing"
	"time"

	. "github.com/onsi/gomega"
	"github.com/tidbcloud/aws-shared-provider/pkg/scheme"
	"github.com/tidbcloud/aws-shared-provider/pkg/tidbmanager"
	"github.com/tidbcloud/aws-shared-provider/pkg/types"
	"github.com/tidbcloud/aws-shared-provider/pkg/utils/clientfake"
	ep "github.com/tidbcloud/aws-shared-provider/pkg/utils/endpoint"
	"github.com/tidbcloud/aws-shared-provider/pkg/utils/kubeapi"
	"github.com/tidbcloud/aws-shared-provider/pkg/utils/labels"
	cloudv1 "github.com/tidbcloud/infra-api/api/v1"
	infra "github.com/tidbcloud/tidb-management-service/api/serverless_infra"
	"github.com/tidbcloud/tidb-management-service/api/serverless_infra/meta"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/record"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"
	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
)

const (
	annoObjectShouldBeDeletedKey    = "ut_check_objectShouldBeDeleted"
	annoNodeShouldNoEvicterTaintKey = "ut_check_nodeShouldNoEvicterTaint"
)

func marshal(v any) []byte {
	b, _ := json.Marshal(v)
	return b
}

func checkNodesAndObjects(g *WithT, kubeClient client.Client, nodes []*corev1.Node, objs []client.Object) {
	for _, n := range nodes {
		g.Expect(kubeClient.Get(context.Background(), client.ObjectKeyFromObject(n), n)).To(Succeed())
		if n.GetAnnotations()[annoNodeShouldNoEvicterTaintKey] != "" {
			g.Expect(n.Spec.Taints).NotTo(
				ContainElement(HaveField("Key", types.ZoneRedundantEvicterTaintKey)),
				"nodeName: %s", n.Name,
			)
		} else {
			if kubeapi.IsNodeUnreachable(n) || !kubeapi.IsNodeReady(n) {
				g.Expect(n.Spec.Taints).NotTo(
					ContainElement(HaveField("Key", types.ZoneRedundantEvicterTaintKey)),
					"nodeName: %s", n.Name,
				)
			} else {
				g.Expect(n.Spec.Taints).To(ContainElement(And(
					HaveField("Key", types.ZoneRedundantEvicterTaintKey),
					HaveField("Effect", corev1.TaintEffectNoSchedule),
				)), "nodeName: %s", n.Name)
			}
		}
	}
	for _, obj := range objs {
		err := kubeClient.Get(context.Background(), client.ObjectKeyFromObject(obj), obj)
		if obj.GetAnnotations()[annoObjectShouldBeDeletedKey] != "" {
			g.Expect(err).To(MatchError(apierrors.IsNotFound, "IsNotFound"),
				"shouldBeDeleted: %s", obj.GetName(),
			)
		} else {
			g.Expect(err).To(Succeed(), obj.GetName(), "objName: %s", obj.GetName())
		}
	}
}

func newFakeReconciler(
	objs []client.Object,
	infraObjs []client.Object,
	tManager map[string]tidbmanager.TidbManager,
	evictionErr error,
) *Reconciler {
	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme.Scheme).
		WithObjects(objs...).
		WithIndex(&corev1.Pod{}, "spec.nodeName", func(obj client.Object) []string {
			pod := obj.(*corev1.Pod)
			return []string{pod.Spec.NodeName}
		}).
		WithIndex(&corev1.Pod{}, "status.unschedulable", func(obj client.Object) []string {
			pod := obj.(*corev1.Pod)
			if kubeapi.IsPodUnschedulable(pod) {
				return []string{"unschedulable"}
			}
			return nil
		}).
		WithInterceptorFuncs(interceptor.Funcs{SubResourceCreate: func(ctx context.Context,
			client client.Client,
			subResourceName string,
			obj client.Object,
			subResource client.Object,
			opts ...client.SubResourceCreateOption,
		) error {
			if evictionErr == nil || subResourceName != "eviction" {
				return client.SubResource(subResourceName).Create(ctx, obj, subResource, opts...)
			} else {
				return evictionErr
			}
		}}).
		Build()
	fakeInfraClient := fake.NewClientBuilder().
		WithScheme(scheme.Scheme).
		WithObjects(infraObjs...).
		WithIndex(&cloudv1.Branch{}, "status.fromProvider.zoneRedundantEnabled", func(obj client.Object) []string {
			clusterStatus, err := obj.(*cloudv1.Branch).AwsFreeTierClusterStatusFromBranch()
			if err != nil {
				return nil
			}
			if clusterStatus.ZoneRedundantEnabled {
				return []string{"zoneRedundantEnabled"}
			}
			return nil
		}).
		Build()

	initClusters := make([]infra.Object, 0, len(infraObjs))
	for _, obj := range infraObjs {
		if cluster, ok := obj.(*cloudv1.Cluster); ok {
			clusterStatus, err := cloudv1.AwsFreeTierClusterStatusFromCluster(cluster)
			if err != nil {
				panic(err)
			}
			zoneRedundantEnabled := "no"
			if clusterStatus.ZoneRedundantEnabled {
				zoneRedundantEnabled = "yes"
			}
			initClusters = append(initClusters, &infra.Cluster{
				ObjectMeta: meta.ObjectMeta{Namespace: cluster.Namespace, Name: cluster.Name},
				Status: infra.ClusterStatus{
					TidbPool:             clusterStatus.TidbPool,
					KeyspaceToken:        clusterStatus.KeyspaceToken,
					ZoneRedundantEnabled: zoneRedundantEnabled,
				},
			})
		}
	}

	return &Reconciler{
		ProviderType: types.ProviderTypeAWSFreeTier,
		Client:       fakeClient,
		InfraClusterCli: clientfake.NewInfraClient(initClusters...).WithIndex(
			"status.zone_redundant_enabled-index",
			func(obj infra.Object) (value, dst any) {
				cluster := obj.(*infra.Cluster)
				return cluster.Status.ZoneRedundantEnabled, meta.ObjectKey{Namespace: cluster.Namespace, Name: cluster.Name}
			},
		).Build(),
		InfraClient:  fakeInfraClient,
		Recorder:     record.NewFakeRecorder(4096),
		TidbManagers: tManager,
	}
}

func TestReconciler_evictZoneRedundant(t *testing.T) {
	tests := []struct {
		name         string
		mainZone     string
		tidbManagers map[string]tidbmanager.TidbManager
		allNodes     []*corev1.Node
		objs         []client.Object
		infraObjs    []client.Object
		evictErr     error
		wantErr      error
	}{
		{
			name:     "evict all pods for complete process",
			mainZone: "primary",
			tidbManagers: map[string]tidbmanager.TidbManager{
				"tidb-pool": &clientfake.FakeTidbManager{
					GetTidbFromFreeCacheR: map[string][]string{"node1": {"tidb-server-fc-pod1"}},
					GetAssignedTidbByTokenR: map[string]*ep.Endpoints{
						"token1": {Endpoints: []string{"tidb-server-assigned-pod1,ip1"}},
					},
				},
			},
			allNodes: []*corev1.Node{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-tidb",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "tidb",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:   corev1.NodeReady,
						Status: corev1.ConditionTrue,
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-gateway",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "critical-services",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-general",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "general",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
			},
			objs: []client.Object{
				&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
					Name: "pvc-tikv-api-evict",
					Annotations: map[string]string{
						types.AnnoVolumeSelectedNode: "node-general",
						annoObjectShouldBeDeletedKey: "1",
					},
				}},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "tikv-api",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
						Labels:      map[string]string{labels.ZREvicterPVCEvictableKey: "true"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-general",
						Volumes: []corev1.Volume{{
							VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
								ClaimName: "pvc-tikv-api-evict",
							}},
						}},
					},
					Status: corev1.PodStatus{
						Phase: corev1.PodPending,
						Conditions: []corev1.PodCondition{{
							Type:   corev1.PodScheduled,
							Reason: corev1.PodReasonUnschedulable,
							Status: corev1.ConditionFalse,
						}},
					},
				},
				&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
					Name: "pvc-load-data-worker-evict",
					Annotations: map[string]string{
						types.AnnoVolumeSelectedNode: "node-tmp-worker",
						annoObjectShouldBeDeletedKey: "1",
					},
				}},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:   "load-data-worker-not-evict",
						Labels: map[string]string{labels.ZREvicterPVCEvictableKey: "true"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tmp-worker",
						Volumes: []corev1.Volume{{
							VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
								ClaimName: "pvc-load-data-worker-evict",
							}},
						}},
					},
					Status: corev1.PodStatus{
						Phase: corev1.PodPending,
						Conditions: []corev1.PodCondition{{
							Type:   corev1.PodScheduled,
							Reason: corev1.PodReasonUnschedulable,
							Status: corev1.ConditionFalse,
						}},
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "tidb-server-fc-pod1",
						Namespace:   "tidb-pool",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tidb",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "tidb-server-assigned-pod1",
						Namespace:   "tidb-pool",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tidb",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "gateway",
						Namespace:   "tidb-admin",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
						Labels:      map[string]string{"control-plane": "tidb-gateway-privatelink-zr"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-gateway",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "tidb-server-other-pod1",
						Namespace:   "tidb-pool",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tidb",
					},
				},
			},
			infraObjs: []client.Object{
				&cloudv1.Cluster{
					ObjectMeta: metav1.ObjectMeta{Name: "cluster-token1", Namespace: "cluster-project1"},
					Status: cloudv1.ClusterStatus{
						FromProvider: runtime.RawExtension{
							Raw: marshal(&cloudv1.AwsFreeTierClusterStatus{
								TidbPool:             "tidb-pool",
								KeyspaceToken:        "token1",
								ZoneRedundantEnabled: true,
							}),
						},
					},
				},
			},
		},
		{
			name:     "evict gateway wait and skip other",
			mainZone: "primary",
			allNodes: []*corev1.Node{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-gateway",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "critical-services",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-tidb",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "tidb",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:   corev1.NodeReady,
						Status: corev1.ConditionTrue,
					}}},
				},
			},
			objs: []client.Object{
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "gateway1",
						Namespace: "tidb-admin",
						OwnerReferences: []metav1.OwnerReference{{
							Controller: ptr.To(true),
							Kind:       "Deployment",
							UID:        "gateway-uid",
						}},
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
						Labels:      map[string]string{"control-plane": "tidb-gateway-privatelink-zr"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-gateway",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "gateway2",
						Namespace: "tidb-admin",
						OwnerReferences: []metav1.OwnerReference{{
							Controller: ptr.To(true),
							Kind:       "Deployment",
							UID:        "gateway-uid",
						}},
						Labels: map[string]string{"control-plane": "tidb-gateway-privatelink-zr"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-gateway",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "tidb-server-fc-pod1",
						Namespace: "tidb-pool",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tidb",
					},
				},
			},
			wantErr: &triggerAfterError{},
		},
		{
			name:     "skip all ignore nodes or components",
			mainZone: "primary",
			allNodes: []*corev1.Node{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other1",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "primary",
							labels.ServerlessComponentNodeLabelKey: "other",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{
							{Key: "not-delete-key1", Effect: corev1.TaintEffectNoSchedule},
							{Key: types.ZoneRedundantEvicterTaintKey, Effect: corev1.TaintEffectNoSchedule},
							{Key: "not-delete-key2", Effect: corev1.TaintEffectNoExecute},
						},
					},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other2",
						Labels: map[string]string{
							labels.ServerlessComponentNodeLabelKey: "other",
						},
					},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other3",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey: "standby",
						},
					},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name:              "node-other4",
						CreationTimestamp: metav1.Now(),
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "other",
						},
					},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name:              "node-other5",
						CreationTimestamp: metav1.Now(),
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "other",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now()},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now()},
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-general",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "general",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now()},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now()},
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-pd",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "pd",
						},
					},
				},
			},
			objs: []client.Object{
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-other1",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other1",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-other2",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other2",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-other3",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other3",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-other4",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other4",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-other5",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other5",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-general",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-general",
					},
				},
				&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
					Name: "pvc-pd-not-evict",
					Annotations: map[string]string{
						types.AnnoVolumeSelectedNode: "node-pd",
					},
				}},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "pod-node-pd",
						Labels: map[string]string{
							labels.ZREvicterIgnoredKey:      "true",
							labels.ZREvicterPVCEvictableKey: "true", // not affect
						},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-pd",
						Volumes: []corev1.Volume{{
							VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
								ClaimName: "pvc-pd-not-evict",
							}},
						}},
					},
					Status: corev1.PodStatus{
						Phase: corev1.PodPending,
						Conditions: []corev1.PodCondition{{
							Type:   corev1.PodScheduled,
							Reason: corev1.PodReasonUnschedulable,
							Status: corev1.ConditionFalse,
						}},
					},
				},
				&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
					Name: "pvc-load-data-worker-not-evict",
					Annotations: map[string]string{
						types.AnnoVolumeSelectedNode: "node-tmp-worker",
					},
				}},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name: "load-data-worker-not-evict",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tmp-worker",
						Volumes: []corev1.Volume{{
							VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
								ClaimName: "pvc-load-data-worker-not-evict",
							}},
						}},
						Tolerations: []corev1.Toleration{{
							Key:      labels.ServerlessComponentNodeLabelKey,
							Operator: corev1.TolerationOpEqual,
							Value:    "tikv",
							Effect:   corev1.TaintEffectNoSchedule,
						}},
					},
					Status: corev1.PodStatus{
						Phase: corev1.PodPending,
						Conditions: []corev1.PodCondition{{
							Type:   corev1.PodScheduled,
							Reason: corev1.PodReasonUnschedulable,
							Status: corev1.ConditionFalse,
						}},
					},
				},
			},
		},
		{
			name:     "evict zr tidb pods, retry",
			mainZone: "primary",
			tidbManagers: map[string]tidbmanager.TidbManager{
				"tidb-pool": &clientfake.FakeTidbManager{
					GetAssignedTidbByTokenR: map[string]*ep.Endpoints{
						"token1": {Endpoints: []string{"tidb-server-assigned-not-evict-pod,ip1"}},
					},
				},
			},
			allNodes: []*corev1.Node{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-tidb",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "tidb",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:   corev1.NodeReady,
						Status: corev1.ConditionTrue,
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-gateway",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "critical-services",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "other",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
			},
			objs: []client.Object{
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "tidb-server-assigned-not-evict-pod",
						Namespace: "tidb-pool",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-tidb",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "gateway",
						Namespace:   "tidb-admin",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
						Labels:      map[string]string{"control-plane": "tidb-gateway-privatelink-zr"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-gateway",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "pod-node-other-not-delete",
						Namespace: "tidb-admin",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other",
					},
				},
			},
			infraObjs: []client.Object{
				&cloudv1.Cluster{
					ObjectMeta: metav1.ObjectMeta{Name: "cluster-token1", Namespace: "cluster-project1"},
					Status: cloudv1.ClusterStatus{
						FromProvider: runtime.RawExtension{
							Raw: marshal(&cloudv1.AwsFreeTierClusterStatus{
								TidbPool:             "tidb-pool",
								KeyspaceToken:        "token1",
								ZoneRedundantEnabled: true,
							}),
						},
					},
				},
			},
			evictErr: apierrors.NewTooManyRequestsError("inject to ut"),
			wantErr:  &triggerAfterError{triggerAfter: RetryTooManyRequestsTime},
		},
		{
			name:     "evict gateway, retry",
			mainZone: "primary",
			allNodes: []*corev1.Node{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-gateway",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "critical-services",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "other",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
			},
			objs: []client.Object{
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "gateway",
						Namespace: "tidb-admin",
						Labels:    map[string]string{"control-plane": "tidb-gateway-privatelink-zr"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-gateway",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "pod-node-other-not-delete",
						Namespace: "tidb-admin",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other",
					},
				},
			},
			evictErr: apierrors.NewTooManyRequestsError("inject to ut"),
			wantErr:  &triggerAfterError{triggerAfter: RetryTooManyRequestsTime},
		},
		{
			name:     "evict other, retry",
			mainZone: "primary",
			allNodes: []*corev1.Node{
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other1",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "other1",
						},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:   corev1.NodeReady,
						Status: corev1.ConditionTrue,
					}}},
				},
				{
					ObjectMeta: metav1.ObjectMeta{
						Name: "node-other2",
						Labels: map[string]string{
							labels.ServerlessZoneLabelKey:          "standby",
							labels.ServerlessComponentNodeLabelKey: "other2",
						},
					},
					Spec: corev1.NodeSpec{
						Taints: []corev1.Taint{{
							Key:       corev1.TaintNodeUnreachable,
							Effect:    corev1.TaintEffectNoExecute,
							TimeAdded: &metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
						}},
					},
					Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{
						Type:               corev1.NodeReady,
						Status:             corev1.ConditionUnknown,
						LastTransitionTime: metav1.Time{Time: time.Now().Add(-LongUnreachableTime)},
					}}},
				},
			},
			objs: []client.Object{
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:      "pod-node-other1-not-evict",
						Namespace: "tidb-admin",
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other1",
					},
				},
				&corev1.Pod{
					ObjectMeta: metav1.ObjectMeta{
						Name:        "pod-node-other2",
						Namespace:   "tidb-admin",
						Annotations: map[string]string{annoObjectShouldBeDeletedKey: "1"},
					},
					Spec: corev1.PodSpec{
						NodeName: "node-other2",
					},
				},
			},
			evictErr: apierrors.NewTooManyRequestsError("inject to ut"),
			wantErr:  &triggerAfterError{triggerAfter: RetryTooManyRequestsTime},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			g := NewGomegaWithT(t)
			allObjs := make([]client.Object, 0, len(tt.objs)+len(tt.allNodes))
			allObjs = append(allObjs, tt.objs...)
			for _, n := range tt.allNodes {
				allObjs = append(allObjs, n)
			}
			r := newFakeReconciler(allObjs, tt.infraObjs, tt.tidbManagers, tt.evictErr)
			err := r.evictZoneRedundant(context.Background(), tt.mainZone)
			if tt.wantErr != nil {
				g.Expect(err).To(MatchError(tt.wantErr))
			} else {
				g.Expect(err).To(Succeed())
			}
			checkNodesAndObjects(g, r.Client, tt.allNodes, tt.objs)
		})
	}
}