package metrics

import (
	"context"
	"testing"

	autoscalingv1beta1 "github.com/crossplane-contrib/provider-aws/apis/autoscaling/v1beta1"
	ec2v1alpha1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1alpha1"
	eksv1alpha1 "github.com/crossplane-contrib/provider-aws/apis/eks/manualv1alpha1"
	gkecontainerv1beta1 "github.com/crossplane-contrib/provider-gcp/apis/container/v1beta1"
	ackv1alpha1 "github.com/crossplane-contrib/provider-upjet-alibabacloud/apis/ack/v1alpha1"
	aliplv1alpha1 "github.com/crossplane-contrib/provider-upjet-alibabacloud/apis/privatelink/v1alpha1"
	xpv1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
	"github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/stretchr/testify/assert"
	ec2v1beta1 "github.com/upbound/provider-aws/apis/ec2/v1beta1"
	upboundeksv1beta2 "github.com/upbound/provider-aws/apis/eks/v1beta2"
	azureksv1beta1 "github.com/upbound/provider-azure/apis/containerservice/v1beta1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/utils/ptr"
	"sigs.k8s.io/controller-runtime/pkg/client"

	agentv2 "github.com/tidbcloud/observability/agent-control-server/api/v2"
	"github.com/tidbcloud/observability/agent-control-server/pkg/consts"
	"github.com/tidbcloud/observability/agent-control-server/pkg/testkits"
)

func TestRecordNodeGroupDesiredCount(t *testing.T) {
	tests := []struct {
		name       string
		vendor     string
		objects    []client.Object
		wantLabels map[string]float64
		wantCount  int
	}{
		{
			name:   "aws vendor records eks nodegroups",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-1"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](3),
							},
						},
					},
				},
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-2"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](5),
							},
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"eks|cluster-a|ng-1|eks.aws.crossplane.io/v1alpha1/NodeGroup": 3,
				"eks|cluster-a|ng-2|eks.aws.crossplane.io/v1alpha1/NodeGroup": 5,
			},
			wantCount: 2,
		},
		{
			name:   "aws vendor aggregates nodegroups and autoscaling groups",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-1"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](3),
							},
						},
					},
				},
				&upboundeksv1beta2.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "upbound-ng"},
					Spec: upboundeksv1beta2.NodeGroupSpec{
						ResourceSpec: xpv1.ResourceSpec{},
						ForProvider: upboundeksv1beta2.NodeGroupParameters{
							ClusterName: ptr.To("cluster-upbound"),
							Region:      ptr.To("us-west-2"),
							ScalingConfig: &upboundeksv1beta2.ScalingConfigParameters{
								DesiredSize: ptr.To[float64](4),
							},
						},
					},
				},
				&autoscalingv1beta1.AutoScalingGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "asg-1"},
					Spec: autoscalingv1beta1.AutoScalingGroupSpec{
						ResourceSpec: xpv1.ResourceSpec{},
						ForProvider: autoscalingv1beta1.AutoScalingGroupParameters{
							Region:          "us-west-2",
							MaxSize:         ptr.To[int64](10),
							MinSize:         ptr.To[int64](1),
							DesiredCapacity: ptr.To[int64](6),
							Tags: []*autoscalingv1beta1.Tag{{
								Key:   ptr.To("eks:cluster-name"),
								Value: ptr.To("cluster-asg"),
							}},
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"eks|cluster-a|ng-1|eks.aws.crossplane.io/v1alpha1/NodeGroup":                  3,
				"eks|cluster-upbound|upbound-ng|eks.aws.upbound.io/v1beta2/NodeGroup":          4,
				"eks|cluster-asg|asg-1|autoscaling.aws.crossplane.io/v1beta1/AutoScalingGroup": 6,
			},
			wantCount: 3,
		},
		{
			name:   "gcp vendor records gke nodepools",
			vendor: consts.VendorTypeGCP,
			objects: []client.Object{
				&gkecontainerv1beta1.NodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-1"},
					Spec: gkecontainerv1beta1.NodePoolSpec{
						ForProvider: gkecontainerv1beta1.NodePoolParameters{
							Cluster:          "cluster-b",
							InitialNodeCount: ptr.To[int64](4),
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"gke|cluster-b|np-1|gcp.NodePool": 4,
			},
			wantCount: 1,
		},
		{
			name:   "aws vendor does not record gke nodepools",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&gkecontainerv1beta1.NodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-ignored"},
					Spec: gkecontainerv1beta1.NodePoolSpec{
						ForProvider: gkecontainerv1beta1.NodePoolParameters{
							Cluster:          "cluster-b",
							InitialNodeCount: ptr.To[int64](10),
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "gcp vendor does not record eks nodegroups",
			vendor: consts.VendorTypeGCP,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-ignored"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](7),
							},
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:      "unknown vendor records nothing",
			vendor:    "unknown",
			wantCount: 0,
		},
		{
			name:      "empty vendor records nothing",
			vendor:    "",
			wantCount: 0,
		},
		{
			name:   "nil scaling config is skipped",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-nil-scaling"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "nil desired size is skipped",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-nil-desired"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName:   "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{},
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "status desired size is used when spec desired size is unset",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-status-fallback"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName:   "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{},
						},
					},
					Status: eksv1alpha1.NodeGroupStatus{
						AtProvider: eksv1alpha1.NodeGroupObservation{
							ScalingConfig: eksv1alpha1.NodeGroupScalingConfigStatus{
								DesiredSize: ptr.To[int32](4),
							},
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"eks|cluster-a|ng-status-fallback|eks.aws.crossplane.io/v1alpha1/NodeGroup": 4,
			},
			wantCount: 1,
		},
		{
			name:   "spec desired size takes precedence over status",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-spec-preferred"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](3),
							},
						},
					},
					Status: eksv1alpha1.NodeGroupStatus{
						AtProvider: eksv1alpha1.NodeGroupObservation{
							ScalingConfig: eksv1alpha1.NodeGroupScalingConfigStatus{
								DesiredSize: ptr.To[int32](9),
							},
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"eks|cluster-a|ng-spec-preferred|eks.aws.crossplane.io/v1alpha1/NodeGroup": 3,
			},
			wantCount: 1,
		},
		{
			name:   "nil initial node count is skipped (gcp)",
			vendor: consts.VendorTypeGCP,
			objects: []client.Object{
				&gkecontainerv1beta1.NodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-nil-count"},
					Spec: gkecontainerv1beta1.NodePoolSpec{
						ForProvider: gkecontainerv1beta1.NodePoolParameters{
							Cluster: "cluster-b",
						},
					},
				},
			},
			wantCount: 0,
		},

		{
			name:   "azure vendor records aks nodepools",
			vendor: consts.VendorTypeAzure,
			objects: []client.Object{
				&azureksv1beta1.KubernetesClusterNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-az-1"},
					Spec: azureksv1beta1.KubernetesClusterNodePoolSpec{
						ForProvider: azureksv1beta1.KubernetesClusterNodePoolParameters{
							NodeCount:           ptr.To[float64](3),
							KubernetesClusterID: ptr.To("/subscriptions/sub-1/resourceGroups/rg/providers/Microsoft.ContainerService/managedClusters/cluster-c"),
						},
					},
				},
				&azureksv1beta1.KubernetesClusterNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-az-2"},
					Spec: azureksv1beta1.KubernetesClusterNodePoolSpec{
						ForProvider: azureksv1beta1.KubernetesClusterNodePoolParameters{
							NodeCount:           ptr.To[float64](7),
							KubernetesClusterID: ptr.To("/subscriptions/sub-1/resourceGroups/rg/providers/Microsoft.ContainerService/managedClusters/cluster-c"),
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"aks|/subscriptions/sub-1/resourceGroups/rg/providers/Microsoft.ContainerService/managedClusters/cluster-c|np-az-1|azure.KubernetesClusterNodePool": 3,
				"aks|/subscriptions/sub-1/resourceGroups/rg/providers/Microsoft.ContainerService/managedClusters/cluster-c|np-az-2|azure.KubernetesClusterNodePool": 7,
			},
			wantCount: 2,
		},
		{
			name:   "alicloud vendor records ack nodepools",
			vendor: consts.VendorTypeAliCloud,
			objects: []client.Object{
				&ackv1alpha1.KubernetesNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-ali-1"},
					Spec: ackv1alpha1.KubernetesNodePoolSpec{
						ForProvider: ackv1alpha1.KubernetesNodePoolParameters{
							DesiredSize: ptr.To("5"),
							ClusterID:   ptr.To("cluster-d"),
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"ack|cluster-d|np-ali-1|alicloud.KubernetesNodePool": 5,
			},
			wantCount: 1,
		},
		{
			name:   "azure vendor does not record eks nodegroups",
			vendor: consts.VendorTypeAzure,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-ignored"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](7),
							},
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "alicloud vendor does not record eks nodegroups",
			vendor: consts.VendorTypeAliCloud,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-ignored"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](7),
							},
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "nil node count is skipped (azure)",
			vendor: consts.VendorTypeAzure,
			objects: []client.Object{
				&azureksv1beta1.KubernetesClusterNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-nil-count"},
					Spec: azureksv1beta1.KubernetesClusterNodePoolSpec{
						ForProvider: azureksv1beta1.KubernetesClusterNodePoolParameters{
							KubernetesClusterID: ptr.To("/subscriptions/sub-1/cluster-c"),
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "nil desired size is skipped (alicloud)",
			vendor: consts.VendorTypeAliCloud,
			objects: []client.Object{
				&ackv1alpha1.KubernetesNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-nil-desired"},
					Spec: ackv1alpha1.KubernetesNodePoolSpec{
						ForProvider: ackv1alpha1.KubernetesNodePoolParameters{
							ClusterID: ptr.To("cluster-d"),
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "invalid desired size string is skipped (alicloud)",
			vendor: consts.VendorTypeAliCloud,
			objects: []client.Object{
				&ackv1alpha1.KubernetesNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-bad-str"},
					Spec: ackv1alpha1.KubernetesNodePoolSpec{
						ForProvider: ackv1alpha1.KubernetesNodePoolParameters{
							DesiredSize: ptr.To("not-a-number"),
							ClusterID:   ptr.To("cluster-d"),
						},
					},
				},
			},
			wantCount: 0,
		},
		{
			name:   "nil cluster id defaults to empty string (azure)",
			vendor: consts.VendorTypeAzure,
			objects: []client.Object{
				&azureksv1beta1.KubernetesClusterNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-no-cluster"},
					Spec: azureksv1beta1.KubernetesClusterNodePoolSpec{
						ForProvider: azureksv1beta1.KubernetesClusterNodePoolParameters{
							NodeCount: ptr.To[float64](2),
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"aks||np-no-cluster|azure.KubernetesClusterNodePool": 2,
			},
			wantCount: 1,
		},
		{
			name:   "nil cluster id defaults to empty string (alicloud)",
			vendor: consts.VendorTypeAliCloud,
			objects: []client.Object{
				&ackv1alpha1.KubernetesNodePool{
					ObjectMeta: metav1.ObjectMeta{Name: "np-no-cluster"},
					Spec: ackv1alpha1.KubernetesNodePoolSpec{
						ForProvider: ackv1alpha1.KubernetesNodePoolParameters{
							DesiredSize: ptr.To("8"),
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"ack||np-no-cluster|alicloud.KubernetesNodePool": 8,
			},
			wantCount: 1,
		},
		{
			name:   "stale metrics are cleared on re-record",
			vendor: consts.VendorTypeAWS,
			objects: []client.Object{
				&eksv1alpha1.NodeGroup{
					ObjectMeta: metav1.ObjectMeta{Name: "ng-fresh"},
					Spec: eksv1alpha1.NodeGroupSpec{
						ForProvider: eksv1alpha1.NodeGroupParameters{
							ClusterName: "cluster-a",
							ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
								DesiredSize: ptr.To[int32](2),
							},
						},
					},
				},
			},
			wantLabels: map[string]float64{
				"eks|cluster-a|ng-fresh|eks.aws.crossplane.io/v1alpha1/NodeGroup": 2,
			},
			wantCount: 1,
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			nodegroupDesiredNodeCount.Reset()

			cli := testkits.BuildClient(tt.objects...)
			err := RecordNodeGroupDesiredCount(cli, tt.vendor)
			assert.NoError(t, err)

			count := testutil.CollectAndCount(nodegroupDesiredNodeCount)
			assert.Equal(t, tt.wantCount, count)

			for key, wantVal := range tt.wantLabels {
				parts := splitMetricKey(key)
				gauge := nodegroupDesiredNodeCount.WithLabelValues(parts[0], parts[1], parts[2], parts[3])
				assert.Equal(t, wantVal, testutil.ToFloat64(gauge))
			}
		})
	}
}

func TestRecordNodeGroupDesiredCount_StaleCleanup(t *testing.T) {
	nodegroupDesiredNodeCount.Reset()

	cli := testkits.BuildClient(
		&eksv1alpha1.NodeGroup{
			ObjectMeta: metav1.ObjectMeta{Name: "ng-old"},
			Spec: eksv1alpha1.NodeGroupSpec{
				ForProvider: eksv1alpha1.NodeGroupParameters{
					ClusterName: "cluster-a",
					ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
						DesiredSize: ptr.To[int32](10),
					},
				},
			},
		},
	)
	err := RecordNodeGroupDesiredCount(cli, consts.VendorTypeAWS)
	assert.NoError(t, err)
	assert.Equal(t, 1, testutil.CollectAndCount(nodegroupDesiredNodeCount))

	cliEmpty := testkits.BuildClient()
	err = RecordNodeGroupDesiredCount(cliEmpty, consts.VendorTypeAWS)
	assert.NoError(t, err)
	assert.Equal(t, 0, testutil.CollectAndCount(nodegroupDesiredNodeCount))
}

func TestRecordNodeGroupDesiredCount_IgnoresMissingAWSResource(t *testing.T) {
	nodegroupDesiredNodeCount.Reset()

	cli := errorListClient{
		Client: testkits.BuildClient(
			&eksv1alpha1.NodeGroup{
				ObjectMeta: metav1.ObjectMeta{Name: "ng-existing"},
				Spec: eksv1alpha1.NodeGroupSpec{
					ForProvider: eksv1alpha1.NodeGroupParameters{
						ClusterName: "cluster-a",
						ScalingConfig: &eksv1alpha1.NodeGroupScalingConfig{
							DesiredSize: ptr.To[int32](2),
						},
					},
				},
			},
		),
		err: apierrors.NewNotFound(schema.GroupResource{Group: "autoscaling.aws.crossplane.io", Resource: "autoscalinggroups"}, "test"),
	}

	err := RecordNodeGroupDesiredCount(cli, consts.VendorTypeAWS)
	assert.NoError(t, err)
	assert.Equal(t, 1, testutil.CollectAndCount(nodegroupDesiredNodeCount))
}

func TestRecordAgentStatus(t *testing.T) {
	agentStatus.Reset()
	agentAnnotations.Reset()
	agentCreated.Reset()

	cli := testkits.BuildClient(
		&agentv2.SharedPoolAgent{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					consts.AnnotationKeyPauseReconcile: "true",
				},
			},
			Spec: agentv2.SharedPoolAgentSpec{
				KubernetesClusterName: "cluster-sp",
				O11y: agentv2.O11yConfigV2{
					TenantID:      "tenant-sp",
					ApplicationID: "app-sp",
				},
			},
			Status: agentv2.SharedPoolAgentStatus{Phase: "Running"},
		},
		&agentv2.TiDBGroupAgent{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					consts.AnnotationKeyO11yVMAgentConfigTemplate: "nextgen/tidb/vmagent.yaml",
				},
			},
			Spec: agentv2.TiDBGroupAgentSpec{
				KubernetesClusterName: "cluster-tg",
				O11y: agentv2.O11yConfigV2{
					TenantID:      "tenant-tg",
					ApplicationID: "app-tg",
				},
			},
			Status: agentv2.TiDBGroupAgentStatus{Phase: "Pending"},
		},
		&agentv2.KubernetesAgent{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					consts.AnnotationKeyO11yVectorConfigTemplate:  "nextgen/k8s/vector.yaml",
					consts.AnnotationKeyO11yVMAgentConfigTemplate: "nextgen/k8s/vmagent.yaml",
				},
			},
			Spec: agentv2.KubernetesAgentSpec{
				KubernetesClusterName: "cluster-k8s",
				O11y: agentv2.O11yConfigV2{
					TenantID:      "tenant-k8s",
					ApplicationID: "app-k8s",
				},
			},
			Status: agentv2.KubernetesAgentStatus{Phase: "Failed"},
		},
		&agentv2.O11yK8sAgent{
			Spec: agentv2.O11yK8sAgentSpec{
				KubernetesClusterName: "cluster-o11yk8s",
				O11y: agentv2.O11yConfig{
					TenantID:      "tenant-o11yk8s",
					ApplicationID: "app-o11yk8s",
				},
			},
			Status: agentv2.O11yK8sAgentStatus{Phase: "Ready"},
		},
		&agentv2.O11yTiDBAgent{
			ObjectMeta: metav1.ObjectMeta{
				Annotations: map[string]string{
					consts.AnnotationKeyPauseReconcile:            "true",
					consts.AnnotationKeyO11yVMAgentConfigTemplate: "dedicated/tidb/vmagent.yaml",
					consts.AnnotationKeyO11yVectorConfigTemplate:  "dedicated/tidb/vector.yaml",
				},
			},
			Spec: agentv2.O11yTiDBAgentSpec{
				KubernetesClusterName: "cluster-o11ytidb",
				O11y: agentv2.O11yConfig{
					TenantID:      "tenant-o11ytidb",
					ApplicationID: "app-o11ytidb",
				},
			},
			Status: agentv2.O11yTiDBAgentStatus{Phase: "Running"},
		},
	)

	err := RecordAgentStatus(cli)
	assert.NoError(t, err)
	assert.Equal(t, 5, testutil.CollectAndCount(agentStatus))
	assert.Equal(t, 5, testutil.CollectAndCount(agentAnnotations))
	assert.Equal(t, 5, testutil.CollectAndCount(agentCreated))

	statusCases := [][]string{
		{consts.AgentSharedPool, "cluster-sp", "tenant-sp", "app-sp", "Running"},
		{consts.AgentTidbGroup, "cluster-tg", "tenant-tg", "app-tg", "Pending"},
		{consts.AgentKubernetes, "cluster-k8s", "tenant-k8s", "app-k8s", "Failed"},
		{consts.AgentO11yK8s, "cluster-o11yk8s", "tenant-o11yk8s", "app-o11yk8s", "Ready"},
		{consts.AgentO11yTidb, "cluster-o11ytidb", "tenant-o11ytidb", "app-o11ytidb", "Running"},
	}
	createdCases := [][]string{
		{consts.AgentSharedPool, "cluster-sp", "tenant-sp", "app-sp"},
		{consts.AgentTidbGroup, "cluster-tg", "tenant-tg", "app-tg"},
		{consts.AgentKubernetes, "cluster-k8s", "tenant-k8s", "app-k8s"},
		{consts.AgentO11yK8s, "cluster-o11yk8s", "tenant-o11yk8s", "app-o11yk8s"},
		{consts.AgentO11yTidb, "cluster-o11ytidb", "tenant-o11ytidb", "app-o11ytidb"},
	}
	for _, lbs := range statusCases {
		assert.Greater(t, testutil.ToFloat64(agentStatus.WithLabelValues(lbs...)), float64(0))
	}
	for _, lbs := range createdCases {
		assert.NotZero(t, testutil.ToFloat64(agentCreated.WithLabelValues(lbs...)))
	}

	annotationCases := []struct {
		lbs []string
	}{
		{[]string{consts.AgentSharedPool, "cluster-sp", "tenant-sp", "app-sp", "true", "-", "-"}},
		{[]string{consts.AgentTidbGroup, "cluster-tg", "tenant-tg", "app-tg", "-", "nextgen/tidb/vmagent.yaml", "-"}},
		{[]string{consts.AgentKubernetes, "cluster-k8s", "tenant-k8s", "app-k8s", "-", "nextgen/k8s/vmagent.yaml", "nextgen/k8s/vector.yaml"}},
		{[]string{consts.AgentO11yK8s, "cluster-o11yk8s", "tenant-o11yk8s", "app-o11yk8s", "-", "-", "-"}},
		{[]string{consts.AgentO11yTidb, "cluster-o11ytidb", "tenant-o11ytidb", "app-o11ytidb", "true", "dedicated/tidb/vmagent.yaml", "dedicated/tidb/vector.yaml"}},
	}
	for _, tc := range annotationCases {
		assert.Greater(t, testutil.ToFloat64(agentAnnotations.WithLabelValues(tc.lbs...)), float64(0))
	}
}

func TestRecordAgentStatus_IgnoreNotFound(t *testing.T) {
	agentStatus.Reset()
	agentAnnotations.Reset()
	agentCreated.Reset()

	cli := selectiveListErrorClient{
		Client: testkits.BuildClient(
			&agentv2.SharedPoolAgent{
				ObjectMeta: metav1.ObjectMeta{
					Annotations: map[string]string{
						consts.AnnotationKeyPauseReconcile: "true",
					},
				},
				Spec: agentv2.SharedPoolAgentSpec{
					KubernetesClusterName: "cluster-sp",
					O11y: agentv2.O11yConfigV2{
						TenantID:      "tenant-sp",
						ApplicationID: "app-sp",
					},
				},
				Status: agentv2.SharedPoolAgentStatus{Phase: "Running"},
			},
		),
		listErr: func(list client.ObjectList) error {
			switch list.(type) {
			case *agentv2.KubernetesAgentList, *agentv2.TiDBGroupAgentList, *agentv2.O11yK8sAgentList, *agentv2.O11yTiDBAgentList:
				return apierrors.NewNotFound(schema.GroupResource{Group: "agent", Resource: "agents"}, "mock")
			default:
				return nil
			}
		},
	}

	err := RecordAgentStatus(cli)
	assert.NoError(t, err)
	assert.Equal(t, 1, testutil.CollectAndCount(agentStatus))
	assert.Greater(t, testutil.ToFloat64(agentStatus.WithLabelValues(
		consts.AgentSharedPool, "cluster-sp", "tenant-sp", "app-sp", "Running",
	)), float64(0))
	assert.Equal(t, 1, testutil.CollectAndCount(agentAnnotations))
	assert.Greater(t, testutil.ToFloat64(agentAnnotations.WithLabelValues(
		consts.AgentSharedPool, "cluster-sp", "tenant-sp", "app-sp",
		"true", "-", "-",
	)), float64(0))
	assert.Equal(t, 1, testutil.CollectAndCount(agentCreated))
	assert.NotZero(t, testutil.ToFloat64(agentCreated.WithLabelValues(
		consts.AgentSharedPool, "cluster-sp", "tenant-sp", "app-sp",
	)))
}

func TestRecordVPCEndpointStatus(t *testing.T) {
	vpcEndpointStatus.Reset()

	ali := &aliplv1alpha1.VpcEndpoint{
		ObjectMeta: metav1.ObjectMeta{Namespace: "ns-ali", Name: "ali-ep"},
		Spec: aliplv1alpha1.VpcEndpointSpec{
			ForProvider: aliplv1alpha1.VpcEndpointParameters{
				ServiceName:  ptr.To("com.aliyun.vpce.svc"),
				VPCID:        ptr.To("vpc-ali"),
				EndpointType: ptr.To("Interface"),
			},
		},
	}
	ali.Status.SetConditions(xpv1.Condition{
		Type:   xpv1.TypeReady,
		Status: corev1.ConditionTrue,
		Reason: xpv1.ConditionReason("Available"),
	})

	awsAlpha := &ec2v1alpha1.VPCEndpoint{
		ObjectMeta: metav1.ObjectMeta{Namespace: "ns-aws-alpha", Name: "aws-alpha-ep"},
		Spec: ec2v1alpha1.VPCEndpointSpec{
			ForProvider: ec2v1alpha1.VPCEndpointParameters{
				Region:          "us-west-2",
				ServiceName:     ptr.To("com.amazonaws.us-west-2.logs"),
				VPCEndpointType: ptr.To("Interface"),
				CustomVPCEndpointParameters: ec2v1alpha1.CustomVPCEndpointParameters{
					VPCID: ptr.To("vpc-aws-alpha"),
				},
			},
		},
	}
	awsAlpha.Status.SetConditions(xpv1.Condition{
		Type:   xpv1.TypeReady,
		Status: corev1.ConditionTrue,
		Reason: xpv1.ConditionReason("Ready"),
	})

	awsBeta := &ec2v1beta1.VPCEndpoint{
		ObjectMeta: metav1.ObjectMeta{Namespace: "ns-aws-beta", Name: "aws-beta-ep"},
		Spec: ec2v1beta1.VPCEndpointSpec{
			ForProvider: ec2v1beta1.VPCEndpointParameters_2{
				Region:          ptr.To("us-west-2"),
				ServiceName:     ptr.To("com.amazonaws.us-west-2.execute-api"),
				VPCID:           ptr.To("vpc-aws-beta"),
				VPCEndpointType: ptr.To("Gateway"),
			},
		},
	}
	awsBeta.Status.SetConditions(xpv1.Condition{
		Type:   xpv1.TypeReady,
		Status: corev1.ConditionUnknown,
		Reason: xpv1.ConditionReason("Reconciling"),
	})

	cli := testkits.BuildClient(ali, awsAlpha, awsBeta)
	err := RecordVPCEndpointStatus(cli)
	assert.NoError(t, err)
	assert.Equal(t, 3, testutil.CollectAndCount(vpcEndpointStatus))

	assert.Greater(t, testutil.ToFloat64(vpcEndpointStatus.WithLabelValues(
		aliplv1alpha1.CRDGroup+"/"+aliplv1alpha1.CRDVersion,
		"ns-ali", "ali-ep", "com.aliyun.vpce.svc", "vpc-ali", "Interface", "Available", "Ready",
	)), float64(0))
	assert.Greater(t, testutil.ToFloat64(vpcEndpointStatus.WithLabelValues(
		ec2v1alpha1.CRDGroup+"/"+ec2v1alpha1.CRDVersion,
		"ns-aws-alpha", "aws-alpha-ep", "com.amazonaws.us-west-2.logs", "vpc-aws-alpha", "Interface", "Ready", "Ready",
	)), float64(0))
	assert.Greater(t, testutil.ToFloat64(vpcEndpointStatus.WithLabelValues(
		ec2v1beta1.CRDGroup+"/"+ec2v1beta1.CRDVersion,
		"ns-aws-beta", "aws-beta-ep", "com.amazonaws.us-west-2.execute-api", "vpc-aws-beta", "Gateway", "Reconciling", "NotReady",
	)), float64(0))
}

func TestExtractAutoScalingGroupClusterName(t *testing.T) {
	tests := []struct {
		name string
		tags []*autoscalingv1beta1.Tag
		want string
	}{
		{
			name: "uses eks cluster-name tag value",
			tags: []*autoscalingv1beta1.Tag{
				{Key: ptr.To("eks:cluster-name"), Value: ptr.To("cluster-a")},
			},
			want: "cluster-a",
		},
		{
			name: "uses kubernetes cluster tag suffix",
			tags: []*autoscalingv1beta1.Tag{
				{Key: ptr.To("kubernetes.io/cluster/cluster-b"), Value: ptr.To("owned")},
			},
			want: "cluster-b",
		},
		{
			name: "handles nil tag and key safely",
			tags: []*autoscalingv1beta1.Tag{
				nil,
				{Value: ptr.To("missing-key")},
			},
			want: "",
		},
		{
			name: "returns empty when no cluster tag exists",
			tags: []*autoscalingv1beta1.Tag{
				{Key: ptr.To("Name"), Value: ptr.To("asg-1")},
			},
			want: "",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert.Equal(t, tt.want, extractAutoScalingGroupClusterName(tt.tags))
		})
	}
}

func splitMetricKey(key string) [4]string {
	var parts [4]string
	idx := 0
	start := 0
	for i, ch := range key {
		if ch == '|' {
			parts[idx] = key[start:i]
			idx++
			start = i + 1
		}
	}
	parts[idx] = key[start:]
	return parts
}

type errorListClient struct {
	client.Client
	err error
}

func (e errorListClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
	switch list.(type) {
	case *upboundeksv1beta2.NodeGroupList, *autoscalingv1beta1.AutoScalingGroupList:
		return e.err
	default:
		return e.Client.List(ctx, list, opts...)
	}
}

type selectiveListErrorClient struct {
	client.Client
	listErr func(list client.ObjectList) error
}

func (s selectiveListErrorClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
	if s.listErr != nil {
		if err := s.listErr(list); err != nil {
			return err
		}
	}
	return s.Client.List(ctx, list, opts...)
}