package clusterupgradeoperation

import (
	"context"
	"encoding/json"
	"errors"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client/fake"

	o11yv1 "github.com/tidbcloud/observability/api/regionalserver/resourceworker/v1"
	"github.com/tidbcloud/observability/pkg/common/constants"
	"github.com/tidbcloud/observability/pkg/resourceworker/controllers/clusterupgradeoperation/providers"
)

func newTestVMCluster(annotations map[string]string) *unstructured.Unstructured {
	vmcluster := newVMClusterObject()
	vmcluster.SetNamespace(constants.ObservabilityNamespace)
	vmcluster.SetName(defaultVMClusterName)
	vmcluster.SetAnnotations(annotations)
	vmcluster.Object["spec"] = map[string]any{"vmstorage": map[string]any{"replicas": float64(1)}}
	return vmcluster
}

func newTestVMStorageHelmValues(t *testing.T) *apiextensionsv1.JSON {
	t.Helper()
	helmValues := map[string]any{"vmstorage": map[string]any{"replicas": 1}}
	helmValuesRaw, err := json.Marshal(helmValues)
	require.NoError(t, err)
	return &apiextensionsv1.JSON{Raw: helmValuesRaw}
}

func TestNormalizeK8sVersion(t *testing.T) {
	tests := []struct {
		name     string
		input    string
		expected string
	}{
		{"trim v prefix", "v1.28.0", "1.28.0"},
		{"trim spaces", "  1.28.0  ", "1.28.0"},
		{"remove vendor suffix", "1.33.3-aliyun.1", "1.33.3"},
		{"combined", " v1.28.0-eks.1 ", "1.28.0"},
		{"empty", "", ""},
		{"only v", "v", ""},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := normalizeK8sVersion(tt.input)
			assert.Equal(t, tt.expected, got)
		})
	}
}

func TestDeepMergeMap(t *testing.T) {
	t.Run("nil value deletes key", func(t *testing.T) {
		dst := map[string]any{"a": "keep", "b": "delete"}
		src := map[string]any{"b": nil}
		deepMergeMap(dst, src)
		assert.Equal(t, map[string]any{"a": "keep"}, dst)
	})

	t.Run("non-map overwrites", func(t *testing.T) {
		dst := map[string]any{"a": "old", "b": 1}
		src := map[string]any{"a": "new", "b": 2}
		deepMergeMap(dst, src)
		assert.Equal(t, "new", dst["a"])
		assert.Equal(t, 2, dst["b"])
	})

	t.Run("nested map merge", func(t *testing.T) {
		dst := map[string]any{"spec": map[string]any{"a": 1, "b": 2}}
		src := map[string]any{"spec": map[string]any{"b": 20, "c": 3}}
		deepMergeMap(dst, src)
		spec := dst["spec"].(map[string]any)
		assert.Equal(t, 1, spec["a"])
		assert.Equal(t, 20, spec["b"])
		assert.Equal(t, 3, spec["c"])
	})

	t.Run("array overwrites", func(t *testing.T) {
		dst := map[string]any{"items": []string{"a"}}
		src := map[string]any{"items": []string{"x", "y"}}
		deepMergeMap(dst, src)
		assert.Equal(t, []string{"x", "y"}, dst["items"])
	})
}

func TestIsStatefulSetRolledOut(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
	r := &ClusterUpgradeOperationReconciler{Client: fakeClient}

	replicas := int32(3)
	tests := []struct {
		name     string
		sts      *appsv1.StatefulSet
		expected bool
	}{
		{
			name: "rolled out",
			sts: &appsv1.StatefulSet{
				ObjectMeta: metav1.ObjectMeta{Name: "sts", Namespace: "ns"},
				Spec:       appsv1.StatefulSetSpec{Replicas: &replicas},
				Status: appsv1.StatefulSetStatus{
					ObservedGeneration: 1,
					Replicas:           3,
					ReadyReplicas:      3,
					UpdatedReplicas:    3,
				},
			},
			expected: true,
		},
		{
			name: "observed gen behind",
			sts: &appsv1.StatefulSet{
				ObjectMeta: metav1.ObjectMeta{Generation: 2},
				Spec:       appsv1.StatefulSetSpec{Replicas: &replicas},
				Status: appsv1.StatefulSetStatus{
					ObservedGeneration: 1,
					UpdatedReplicas:    3,
					ReadyReplicas:      3,
				},
			},
			expected: false,
		},
		{
			name: "updated replicas not ready",
			sts: &appsv1.StatefulSet{
				Spec: appsv1.StatefulSetSpec{Replicas: &replicas},
				Status: appsv1.StatefulSetStatus{
					ObservedGeneration: 1,
					UpdatedReplicas:    2,
					ReadyReplicas:      3,
				},
			},
			expected: false,
		},
		{
			name: "ready replicas less than spec",
			sts: &appsv1.StatefulSet{
				Spec: appsv1.StatefulSetSpec{Replicas: &replicas},
				Status: appsv1.StatefulSetStatus{
					ObservedGeneration: 1,
					UpdatedReplicas:    3,
					ReadyReplicas:      2,
				},
			},
			expected: false,
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := r.isStatefulSetRolledOut(tt.sts)
			assert.Equal(t, tt.expected, got)
		})
	}
}

func TestHasClusterUpgradeAnnotation(t *testing.T) {
	scheduleTime := time.Now().Add(-1 * time.Hour)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			Schedule: metav1.Time{Time: scheduleTime},
		},
	}

	t.Run("no annotations", func(t *testing.T) {
		assert.False(t, hasClusterUpgradeAnnotation(nil, op))
	})

	t.Run("has LastUpdatedBy matching op name", func(t *testing.T) {
		annotations := map[string]string{
			o11yv1.ClusterUpgradeAnnotationLastUpdatedBy: "op-1",
		}
		assert.True(t, hasClusterUpgradeAnnotation(annotations, op))
	})

	t.Run("LastUpdatedAt after schedule", func(t *testing.T) {
		annotations := map[string]string{
			o11yv1.ClusterUpgradeAnnotationLastUpdatedAt: time.Now().Add(-30 * time.Minute).Format(time.RFC3339),
		}
		assert.True(t, hasClusterUpgradeAnnotation(annotations, op))
	})

	t.Run("LastUpdatedAt before schedule", func(t *testing.T) {
		annotations := map[string]string{
			o11yv1.ClusterUpgradeAnnotationLastUpdatedAt: scheduleTime.Add(-1 * time.Hour).Format(time.RFC3339),
		}
		assert.False(t, hasClusterUpgradeAnnotation(annotations, op))
	})
}

func TestK8sVersionsEqual(t *testing.T) {
	tests := []struct {
		name     string
		target   string
		current  string
		expected bool
	}{
		// x.y: compare first 2 parts
		{
			name:     "x.y - target 1.28 matches current 1.28.0",
			target:   "1.28",
			current:  "1.28.0",
			expected: true,
		},
		{
			name:     "x.y - target 1.28 matches current 1.28.5",
			target:   "1.28",
			current:  "1.28.5",
			expected: true,
		},
		{
			name:     "x.y - target 1.28 does not match current 1.29.0",
			target:   "1.28",
			current:  "1.29.0",
			expected: false,
		},
		{
			name:     "x.y - target 1.28 does not match current 1.27.0",
			target:   "1.28",
			current:  "1.27.0",
			expected: false,
		},
		{
			name:     "x.y - with v prefix",
			target:   "v1.28",
			current:  "1.28.0",
			expected: true,
		},
		{
			name:     "x.y - current has only 1 part",
			target:   "1.28",
			current:  "1",
			expected: false,
		},

		// x.y.z: compare first 3 parts
		{
			name:     "x.y.z - exact match",
			target:   "1.28.0",
			current:  "1.28.0",
			expected: true,
		},
		{
			name:     "x.y.z - target 1.28.0 matches current 1.28.0-aliyun.1 (current has suffix)",
			target:   "1.28.0",
			current:  "1.28.0-aliyun.1",
			expected: true,
		},
		{
			name:     "x.y.z - target 1.28.0 does not match current 1.28.1",
			target:   "1.28.0",
			current:  "1.28.1",
			expected: false,
		},
		{
			name:     "x.y.z - target 1.28.0 does not match current 1.29.0",
			target:   "1.28.0",
			current:  "1.29.0",
			expected: false,
		},
		{
			name:     "x.y.z - current has only 2 parts",
			target:   "1.28.0",
			current:  "1.28",
			expected: false,
		},

		// x.y.z-xxx: compare full strings
		{
			name:     "x.y.z-xxx - exact match",
			target:   "1.28.0-aliyun.1",
			current:  "1.28.0-aliyun.1",
			expected: true,
		},
		{
			name:     "x.y.z-xxx - suffix differs",
			target:   "1.28.0-aliyun.1",
			current:  "1.28.0-aliyun.2",
			expected: false,
		},
		{
			name:     "x.y.z-xxx - same base version different suffix",
			target:   "1.28.0-aliyun.1",
			current:  "1.28.0",
			expected: false,
		},
		{
			name:     "x.y.z-xxx - with v prefix",
			target:   "v1.28.0-aliyun.1",
			current:  "1.28.0-aliyun.1",
			expected: true,
		},

		// edge cases
		{
			name:     "empty target and current",
			target:   "",
			current:  "",
			expected: true,
		},
		{
			name:     "empty target",
			target:   "",
			current:  "1.28.0",
			expected: false,
		},
		{
			name:     "empty current",
			target:   "1.28",
			current:  "",
			expected: false,
		},
		{
			name:     "single part target - exact match",
			target:   "1",
			current:  "1",
			expected: true,
		},
		{
			name:     "single part target - no match",
			target:   "1",
			current:  "1.28.0",
			expected: false,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := k8sVersionsEqual(tt.target, tt.current)
			assert.Equal(t, tt.expected, got, "target=%q current=%q", tt.target, tt.current)
		})
	}
}

func TestAddonVersionsEqual(t *testing.T) {
	tests := []struct {
		name     string
		target   string
		current  string
		expected bool
	}{
		{
			name:     "same eksbuild version",
			target:   "v1.12.0-eksbuild.3",
			current:  "v1.12.0-eksbuild.3",
			expected: true,
		},
		{
			name:     "different eksbuild version",
			target:   "v1.12.0-eksbuild.3",
			current:  "v1.12.0-eksbuild.1",
			expected: false,
		},
		{
			name:     "ignore optional v prefix",
			target:   "1.12.0-eksbuild.3",
			current:  "v1.12.0-eksbuild.3",
			expected: true,
		},
		{
			name:     "trim spaces",
			target:   " v1.12.0-eksbuild.3 ",
			current:  "v1.12.0-eksbuild.3",
			expected: true,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := addonVersionsEqual(tt.target, tt.current)
			assert.Equal(t, tt.expected, got)
		})
	}
}

func TestProcessUpgradeNodeGroup_UnknownStrategy(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      "unknown-strategy",
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	// Mock provider must return a nodegroup so we reach the strategy switch
	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{{ID: "ng-1", Name: "ng-1", Status: providers.NodeGroupStatusActive}},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeNodeGroup_CloudProviderError(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	// Empty registry - Get will return "unsupported cloud provider"
	registry := providers.NewCloudProviderRegistryForTesting(nil)

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeControlPlane_AlreadyAtTarget(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ControlPlaneUpgradeOptions{TargetVersion: "1.28"}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ControlPlaneVersion: "1.28.0", // already at target
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeControlPlane(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep) // step finished
}

func TestProcessUpgradeControlPlane_Updating(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ControlPlaneUpgradeOptions{TargetVersion: "1.28"}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ControlPlaneStatus: providers.ControlPlaneStatusUpdating,
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeControlPlane(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessUpgradeClusterAddons_AllDone(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{} // default returns ACTIVE with target version
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep)
}

func TestProcessUpgradeClusterAddons_Updating(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				// BeginTime recent so we don't hit 30min timeout before UPDATING requeue path
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ClusterAddonsStatus: []providers.AddonStatusInfo{{Name: "vpc-cni", Version: "1.1.0", Status: "UPDATING"}},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessUpgradeControlPlane_UnmarshalError(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	// Use valid JSON that fails to unmarshal into ControlPlaneUpgradeOptions (e.g. array instead of object)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: []byte(`[]`)},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: &MockCloudProvider{},
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeControlPlane(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeControlPlane_Timeout(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ControlPlaneUpgradeOptions{TargetVersion: "1.28"}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	// BeginTime 50 minutes ago - past the 45 min timeout
	beginTime := time.Now().Add(-50 * time.Minute)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: beginTime},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{ControlPlaneVersion: "1.27.0"} // not at target yet
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeControlPlane(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeControlPlane_StartsUpgrade(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ControlPlaneUpgradeOptions{TargetVersion: "1.28"}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-5 * time.Minute)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{ControlPlaneVersion: "1.27.0"} // not at target, will trigger upgrade
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeControlPlane(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessUpgradeControlPlane_UpgradeControlPlaneErr(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ControlPlaneUpgradeOptions{TargetVersion: "1.28"}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-5 * time.Minute)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ControlPlaneVersion:    "1.27.0",
		UpgradeControlPlaneErr: errors.New("upgrade failed"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeControlPlane(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "failed to upgrade control plane")
	assert.Equal(t, ctrl.Result{}, res)
}

func TestProcessUpgradeClusterAddons_UnmarshalError(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	// Use valid JSON that fails to unmarshal into ClusterAddonsUpgradeOptions (e.g. array instead of object)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: []byte(`[]`)},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: &MockCloudProvider{},
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeClusterAddons_AddonFailure(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ClusterAddonsStatus: []providers.AddonStatusInfo{{Name: "vpc-cni", Version: "1.1.0", Status: "UPDATE_FAILED"}},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeClusterAddons_Timeout(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	beginTime := time.Now().Add(-35 * time.Minute) // past 30 min timeout
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: beginTime},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ClusterAddonsStatus: []providers.AddonStatusInfo{{Name: "vpc-cni", Version: "1.1.0", Status: "UPDATING"}},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeNodeGroup_UnmarshalError(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	// Use valid JSON that fails to unmarshal into NodeGroupUpgradeOptions (e.g. array instead of object)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: []byte(`[]`)},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: &MockCloudProvider{},
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessUpgradeNodeGroup_NodeGroupNotFound_BlueGreenDeleteStep(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-old",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepDeleteOldNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	// Empty NodeGroups - GetNodeGroupByName returns ErrNodeGroupNotFound
	mockProvider := &MockCloudProvider{NodeGroups: []providers.NodeGroup{}}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep)
}

func TestProcessInPlaceUpgrade_FailureState(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusCreateFailed},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessInPlaceUpgrade_CloudOperationIDDone(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor:        o11yv1.ProcessorUpgradeNodeGroup,
				Options:          runtime.RawExtension{Raw: optsRaw},
				CloudOperationID: "op-123",
				BeginTime:        &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	// MockCloudProvider.GetNodeGroupUpgradeOperationStatus returns (true, nil) = done
	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep)
}

func TestProcessInPlaceUpgrade_UpdatingStatus(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusUpdating},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessInPlaceUpgrade_Timeout(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	beginTime := time.Now().Add(-185 * time.Minute) // past 180 min timeout
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: beginTime},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	// Use Active status (not Updating) so we reach the timeout check at step 5
	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessInPlaceUpgrade_AlreadyAtTarget(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep)
}

// ========== BlueGreen NodeGroup Upgrade Tests ==========

func TestProcessBlueGreen_CreateNodeGroup(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCreateNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive, MinSize: 1, MaxSize: 3, DesiredSize: 3},
		},
		CreateNodeGroupAddsToNodeGroups: true,
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepWaitReady, op.Status.CurrentStep.SubStep)
	require.Len(t, mockProvider.CreatedNodeGroupSpecs, 1)
	assert.Equal(t, 3, mockProvider.CreatedNodeGroupSpecs[0].MinSize)
}

func TestProcessBlueGreen_WaitReady_NewNgActive(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepWaitReady,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepCordon, op.Status.CurrentStep.SubStep)
}

func TestProcessBlueGreen_WaitReady_NewNgCreateFailed(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepWaitReady,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusCreateFailed},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessBlueGreen_Cordon_SkipsVMStorage_GoesToDelete(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	node := &corev1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name:   "node-1",
			Labels: map[string]string{"node-group": "ng-old"},
		},
		Spec: corev1.NodeSpec{Unschedulable: false},
	}

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCordon,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, node).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepDeleteOldNodeGroup, op.Status.CurrentStep.SubStep)
}

func TestProcessBlueGreen_Cordon_RequiresVMStorageRollout_GoesToRollout(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	node := &corev1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name:   "node-1",
			Labels: map[string]string{"node-group": "ng-old"},
		},
	}

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:            "ng-old",
		NewNodeGroupName:         "ng-new",
		Strategy:                 o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		RequiresVMStorageRollout: true,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCordon,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, node).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepVMStorageRollout, op.Status.CurrentStep.SubStep)
}

func TestProcessBlueGreen_VMStorageRollout_STSNameEmpty_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:            "ng-old",
		NewNodeGroupName:         "ng-new",
		Strategy:                 o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		RequiresVMStorageRollout: true,
		VMStorageSTSName:         "",
		VMStorageSTSNamespace:    "observability",
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessBlueGreen_DeleteOldNodeGroup_Active_TriggersDelete(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepDeleteOldNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_DeleteOldNodeGroup_DeleteFailed(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepDeleteOldNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
		GetNodeGroupStatusOverride: map[string]providers.NodeGroupStatus{
			"ng-old": providers.NodeGroupStatusDeleteFailed,
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessBlueGreen_DeleteOldNodeGroup_Deleting_Requeues(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepDeleteOldNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
		GetNodeGroupStatusOverride: map[string]providers.NodeGroupStatus{
			"ng-old": providers.NodeGroupStatusDeleting,
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_VMStorageRollout_Success(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{},
		},
		Status: appsv1.StatefulSetStatus{
			ObservedGeneration: 1,
			Replicas:           1,
			ReadyReplicas:      1,
			UpdatedReplicas:    1,
		},
	}

	node := &corev1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name:   "node-ng-new-1",
			Labels: map[string]string{"node-group": "ng-new"},
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "vmstorage-0",
			Namespace: "observability",
			Labels:    map[string]string{"app": "vmstorage"},
		},
		Spec: corev1.PodSpec{NodeName: "node-ng-new-1"},
	}
	vmcluster := newTestVMCluster(map[string]string{
		o11yv1.ClusterUpgradeAnnotationLastUpdatedBy: "op-1",
	})

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   newTestVMStorageHelmValues(t),
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	scheduleTime := time.Now().Add(-2 * time.Hour)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: scheduleTime},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, node, pod, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepDeleteOldNodeGroup, op.Status.CurrentStep.SubStep)
}

func TestProcessBlueGreen_Cordon_Proceeds(t *testing.T) {
	// Cordon proceeds when both ng-old and ng-new exist (GetNodeGroupByName must succeed for ng-new)
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	node := &corev1.Node{
		ObjectMeta: metav1.ObjectMeta{Name: "node-1", Labels: map[string]string{"node-group": "ng-old"}},
		Spec:       corev1.NodeSpec{Unschedulable: false},
	}
	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCordon,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, node).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepDeleteOldNodeGroup, op.Status.CurrentStep.SubStep)
}

func TestProcessBlueGreen_VMStorageRollout_NoVMClusterHelmValues_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: nil,
				},
			},
		},
		Status: appsv1.StatefulSetStatus{
			ObservedGeneration: 1,
			Replicas:           1,
			ReadyReplicas:      1,
			UpdatedReplicas:    1,
		},
	}
	vmcluster := newTestVMCluster(nil)

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	scheduleTime := time.Now().Add(-2 * time.Hour)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: scheduleTime},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "direct StatefulSet patching is not supported")
	assert.Equal(t, ctrl.Result{}, res)
}

func TestProcessBlueGreen_VMStorageRollout_RolloutInProgress(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(3)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{},
		},
		Status: appsv1.StatefulSetStatus{
			ObservedGeneration: 1,
			Replicas:           3,
			ReadyReplicas:      2,
			UpdatedReplicas:    2,
		},
	}
	vmcluster := newTestVMCluster(map[string]string{
		o11yv1.ClusterUpgradeAnnotationLastUpdatedBy: "op-1",
	})

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   newTestVMStorageHelmValues(t),
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	scheduleTime := time.Now().Add(-2 * time.Hour)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: scheduleTime},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_VMStorageRollout_PodsNotOnNewNodes(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{},
		},
		Status: appsv1.StatefulSetStatus{
			ObservedGeneration: 1,
			Replicas:           1,
			ReadyReplicas:      1,
			UpdatedReplicas:    1,
		},
	}

	nodeOld := &corev1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name:   "node-ng-old-1",
			Labels: map[string]string{"node-group": "ng-old"},
		},
	}
	pod := &corev1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "vmstorage-0",
			Namespace: "observability",
			Labels:    map[string]string{"app": "vmstorage"},
		},
		Spec: corev1.PodSpec{NodeName: "node-ng-old-1"},
	}
	vmcluster := newTestVMCluster(map[string]string{
		o11yv1.ClusterUpgradeAnnotationLastUpdatedBy: "op-1",
	})

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   newTestVMStorageHelmValues(t),
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	scheduleTime := time.Now().Add(-2 * time.Hour)
	stepTime := time.Now().Add(-nodeGroupUpgradeTimeout - time.Minute)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: scheduleTime},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: stepTime},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, nodeOld, pod, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessBlueGreen_CreateNodeGroup_CreateFails_Retries(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCreateNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
		CreateNodeGroupErr: errors.New("cloud API temporarily unavailable"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, 3*interval, res.RequeueAfter)
}

func TestProcessBlueGreen_VMStorageRollout_VMClusterNotFound_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{Annotations: nil},
			},
		},
	}

	helmValues := map[string]any{"vmstorage": map[string]any{"replicas": 3}}
	helmValuesRaw, err := json.Marshal(helmValues)
	require.NoError(t, err)

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   &apiextensionsv1.JSON{Raw: helmValuesRaw},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "get VMCluster")
	assert.Equal(t, ctrl.Result{}, res)
}

func TestProcessBlueGreen_VMStorageRollout_InvalidVMClusterHelmValues_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{Annotations: nil},
			},
		},
		Status: appsv1.StatefulSetStatus{
			ObservedGeneration: 1,
			CurrentRevision:    "rev-1",
			UpdateRevision:     "rev-1",
			ReadyReplicas:      1,
			UpdatedReplicas:    1,
		},
	}
	vmcluster := newTestVMCluster(nil)

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   &apiextensionsv1.JSON{Raw: []byte("[]")},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "invalid vmClusterHelmValues")
	assert.Equal(t, ctrl.Result{}, res)
}

func TestProcessBlueGreen_VMStorageRollout_WithVMClusterHelmValues(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	vmcluster := &unstructured.Unstructured{}
	vmcluster.SetGroupVersionKind(schema.GroupVersionKind{
		Group:   "operator.victoriametrics.com",
		Version: "v1beta1",
		Kind:    "VMCluster",
	})
	vmcluster.SetNamespace(constants.ObservabilityNamespace)
	vmcluster.SetName("vmcluster")
	vmcluster.Object["spec"] = map[string]any{"vmstorage": map[string]any{"replicas": float64(3)}}

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{Annotations: nil},
			},
		},
	}

	helmValues := map[string]any{"vmstorage": map[string]any{"replicas": 3}}
	helmValuesRaw, err := json.Marshal(helmValues)
	require.NoError(t, err)

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   &apiextensionsv1.JSON{Raw: helmValuesRaw},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	scheduleTime := time.Now().Add(-2 * time.Hour)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: scheduleTime},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_VMStorageRollout_WithEmptyVMClusterHelmValues(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	vmcluster := newTestVMCluster(nil)
	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
		},
	}

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   &apiextensionsv1.JSON{Raw: []byte("{}")},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: time.Now().Add(-2 * time.Hour)},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	require.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)

	updated := newVMClusterObject()
	require.NoError(t, fakeClient.Get(context.Background(), types.NamespacedName{
		Namespace: constants.ObservabilityNamespace,
		Name:      defaultVMClusterName,
	}, updated))
	assert.Equal(t, "op-1", updated.GetAnnotations()[o11yv1.ClusterUpgradeAnnotationLastUpdatedBy])
	spec, found, err := unstructured.NestedMap(updated.Object, "spec")
	require.NoError(t, err)
	require.True(t, found)
	vmstorageSpec, ok := spec["vmstorage"].(map[string]any)
	require.True(t, ok)
	assert.EqualValues(t, 1, vmstorageSpec["replicas"])
}

func TestProcessUpgradeNodeGroup_UnknownStrategy_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      "unknown-strategy",
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessBlueGreen_UnknownSubStep_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   "unknown-substep",
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

// ========== Additional tests for 85% coverage ==========

func TestProcessInPlaceUpgrade_CloudOperationIDInProgress(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor:        o11yv1.ProcessorUpgradeNodeGroup,
				Options:          runtime.RawExtension{Raw: optsRaw},
				CloudOperationID: "op-123",
				BeginTime:        &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
		GetNodeGroupUpgradeOperationStatusInProgress: true,
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessInPlaceUpgrade_CloudOperationIDError(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor:        o11yv1.ProcessorUpgradeNodeGroup,
				Options:          runtime.RawExtension{Raw: optsRaw},
				CloudOperationID: "op-123",
				BeginTime:        &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
		GetNodeGroupUpgradeOperationStatusErr: errors.New("operation failed"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}

func TestProcessInPlaceUpgrade_NodeGroupCreating_Waits(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusCreating},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessInPlaceUpgrade_TargetRuntimeVersion(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:        "ng-1",
		Strategy:             o11yv1.NodeGroupUpgradeStrategyInPlace,
		TargetRuntimeVersion: "1.28.0",
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now().Add(-1 * time.Hour)},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.28.0", RuntimeVersion: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep)
}

func TestProcessInPlaceUpgrade_UpgradeNodeGroupReturnsOperationID(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
		UpgradeNodeGroupOperationID: "gke-op-123",
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, "gke-op-123", op.Status.CurrentStep.CloudOperationID)
}

func TestProcessInPlaceUpgrade_UpgradeNodeGroupFails_Retries(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName: "ng-1",
		Strategy:      o11yv1.NodeGroupUpgradeStrategyInPlace,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-1", Name: "ng-1", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
		UpgradeNodeGroupErr: errors.New("quota exceeded"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessUpgradeClusterAddons_Creating(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ClusterAddonsStatus: []providers.AddonStatusInfo{{Name: "vpc-cni", Version: "1.1.0", Status: "CREATING"}},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessUpgradeClusterAddons_TriggersUpgrade(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ClusterAddonsStatus: []providers.AddonStatusInfo{{Name: "vpc-cni", Version: "1.1.0", Status: "ACTIVE"}},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessUpgradeClusterAddons_UpgradeClusterAddonsErr(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ClusterAddonsUpgradeOptions{
		Addons: []o11yv1.AddonUpgrade{{Name: "vpc-cni", Version: "1.2.0"}},
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeClusterAddons,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ClusterAddonsStatus:     []providers.AddonStatusInfo{{Name: "vpc-cni", Version: "1.1.0", Status: "ACTIVE"}},
		UpgradeClusterAddonsErr: errors.New("upgrade addons failed"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeClusterAddons(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "failed to upgrade cluster addons")
	assert.Equal(t, ctrl.Result{}, res)
}

func TestProcessUpgradeControlPlane_GetControlPlaneStatusError(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.ControlPlaneUpgradeOptions{TargetVersion: "1.28"}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeControlPlane,
				Options:   runtime.RawExtension{Raw: optsRaw},
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		ControlPlaneVersion:      "1.27.0",
		GetControlPlaneStatusErr: errors.New("API unavailable"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	_, err = r.processUpgradeControlPlane(context.Background(), op)
	assert.Error(t, err)
}

func TestProcessBlueGreen_Cordon_GetLabelSelectorErr(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCordon,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
		GetNodeGroupLabelSelectorErr: errors.New("label selector error"),
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "failed to cordon nodegroup")
	assert.Equal(t, ctrl.Result{}, res)
}

func TestProcessBlueGreen_Cordon_NewNgNotActive_Waits(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCordon,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusCreating},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_DeleteOldNodeGroup_GetStatusErr_Finishes(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepDeleteOldNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive, MinSize: 3},
		},
		GetNodeGroupStatusErr: map[string]error{"ng-old": providers.ErrNodeGroupNotFound},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Nil(t, op.Status.CurrentStep)
	assert.Equal(t, map[string]int{"ng-new": 1}, mockProvider.UpdatedNodeGroupMinSizes)
}

func TestProcessBlueGreen_CreateNodeGroup_WithLaunchTemplate(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCreateNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{
				ID:                    "ng-old",
				Name:                  "ng-old",
				Version:               "1.27.0",
				Status:                providers.NodeGroupStatusActive,
				InstanceType:          []string{"m5.large"},
				AmiType:               "BOTTLEROCKET_x86_64",
				DiskSize:              80,
				LaunchTemplateId:      "lt-123",
				LaunchTemplateVersion: "1",
			},
		},
		CreateNodeGroupAddsToNodeGroups: true,
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepWaitReady, op.Status.CurrentStep.SubStep)
	require.Len(t, mockProvider.CreatedNodeGroupSpecs, 1)
	assert.Equal(t, "lt-123", mockProvider.CreatedNodeGroupSpecs[0].LaunchTemplateId)
	assert.Equal(t, "1", mockProvider.CreatedNodeGroupSpecs[0].LaunchTemplateVersion)
	assert.Equal(t, []string{"m5.large"}, mockProvider.CreatedNodeGroupSpecs[0].InstanceType)
	assert.Equal(t, "BOTTLEROCKET_x86_64", mockProvider.CreatedNodeGroupSpecs[0].AmiType)
	assert.Equal(t, 80, mockProvider.CreatedNodeGroupSpecs[0].DiskSize)
}

func TestProcessBlueGreen_CreateNodeGroup_WithLaunchTemplateCustomAMI_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepCreateNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{
				ID:                    "ng-old",
				Name:                  "ng-old",
				Version:               "1.27.0",
				Status:                providers.NodeGroupStatusActive,
				InstanceType:          []string{"r6a.xlarge"},
				LaunchTemplateId:      "lt-123",
				LaunchTemplateVersion: "1",
			},
		},
		CreateNodeGroupErr: providers.ErrNodeGroupLaunchTemplateCustomAMI,
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
	assert.Empty(t, mockProvider.CreatedNodeGroupSpecs)
}

func TestProcessBlueGreen_GetNodeGroupByNameErr(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepWaitReady,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
		},
		GetNodeGroupByNameErr: map[string]error{"ng-new": errors.New("api error")},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.Error(t, err)
	assert.Contains(t, err.Error(), "failed to get nodegroup by name")
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_DeleteOldNodeGroup_Proceeds(t *testing.T) {
	// DeleteOldNodeGroup proceeds when both ng-old and ng-new exist (GetNodeGroupByName must succeed for ng-new)
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepDeleteOldNodeGroup,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
}

func TestProcessBlueGreen_WaitReady_NewNgCreating_Waits(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:    "ng-old",
		NewNodeGroupName: "ng-new",
		Strategy:         o11yv1.NodeGroupUpgradeStrategyBlueGreen,
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepWaitReady,
				BeginTime: &metav1.Time{Time: time.Now()},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusCreating},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.NotZero(t, res.RequeueAfter)
	assert.Equal(t, o11yv1.BlueGreenSubStepWaitReady, op.Status.CurrentStep.SubStep)
}

func TestProcessBlueGreen_VMStorageRollout_EmptyPodsReplicasPositive_Fails(t *testing.T) {
	scheme := runtime.NewScheme()
	require.NoError(t, appsv1.AddToScheme(scheme))
	require.NoError(t, corev1.AddToScheme(scheme))
	require.NoError(t, o11yv1.AddToScheme(scheme))

	replicas := int32(1)
	sts := &appsv1.StatefulSet{
		ObjectMeta: metav1.ObjectMeta{Name: "vmstorage", Namespace: "observability"},
		Spec: appsv1.StatefulSetSpec{
			Replicas: &replicas,
			Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "vmstorage"}},
			Template: corev1.PodTemplateSpec{},
		},
		Status: appsv1.StatefulSetStatus{
			ObservedGeneration: 1,
			Replicas:           1,
			ReadyReplicas:      1,
			UpdatedReplicas:    1,
		},
	}
	nodeNew := &corev1.Node{
		ObjectMeta: metav1.ObjectMeta{
			Name:   "node-ng-new-1",
			Labels: map[string]string{"node-group": "ng-new"},
		},
	}
	vmcluster := newTestVMCluster(map[string]string{
		o11yv1.ClusterUpgradeAnnotationLastUpdatedBy: "op-1",
	})
	// No pods - List will return empty, arePodsOnNewNodes returns false (replicas>0 but no pods)
	opts := o11yv1.NodeGroupUpgradeOptions{
		NodeGroupName:         "ng-old",
		NewNodeGroupName:      "ng-new",
		Strategy:              o11yv1.NodeGroupUpgradeStrategyBlueGreen,
		VMStorageSTSName:      "vmstorage",
		VMStorageSTSNamespace: "observability",
		VMClusterHelmValues:   newTestVMStorageHelmValues(t),
	}
	optsRaw, err := json.Marshal(opts)
	require.NoError(t, err)

	scheduleTime := time.Now().Add(-2 * time.Hour)
	stepTime := time.Now().Add(-nodeGroupUpgradeTimeout - time.Minute)
	op := &o11yv1.ClusterUpgradeOperation{
		ObjectMeta: metav1.ObjectMeta{Name: "op-1", Namespace: "default"},
		Spec: o11yv1.ClusterUpgradeOperationSpec{
			K8sClusterName: "cluster-1",
			CloudProvider:  o11yv1.CloudProviderEKS,
			TargetVersion:  "1.28",
			Schedule:       metav1.Time{Time: scheduleTime},
		},
		Status: o11yv1.ClusterUpgradeOperationStatus{
			Phase: o11yv1.ClusterUpgradeOperationPhaseRunning,
			CurrentStep: &o11yv1.UpgradeStep{
				Processor: o11yv1.ProcessorUpgradeNodeGroup,
				Options:   runtime.RawExtension{Raw: optsRaw},
				SubStep:   o11yv1.BlueGreenSubStepVMStorageRollout,
				BeginTime: &metav1.Time{Time: stepTime},
			},
		},
	}

	fakeClient := fake.NewClientBuilder().
		WithScheme(scheme).
		WithObjects(op, sts, nodeNew, vmcluster).
		WithStatusSubresource(op).
		Build()

	mockProvider := &MockCloudProvider{
		NodeGroups: []providers.NodeGroup{
			{ID: "ng-old", Name: "ng-old", Version: "1.27.0", Status: providers.NodeGroupStatusActive},
			{ID: "ng-new", Name: "ng-new", Version: "1.28.0", Status: providers.NodeGroupStatusActive},
		},
	}
	registry := providers.NewCloudProviderRegistryForTesting(map[o11yv1.CloudProvider]providers.CloudProviderInterface{
		o11yv1.CloudProviderEKS: mockProvider,
	})

	r := &ClusterUpgradeOperationReconciler{
		Client:                fakeClient,
		cloudProviderRegistry: registry,
	}

	res, err := r.processUpgradeNodeGroup(context.Background(), op)
	assert.NoError(t, err)
	assert.Equal(t, ctrl.Result{}, res)
	assert.Equal(t, o11yv1.ClusterUpgradeOperationPhaseFailed, op.Status.Phase)
}