package cluster import ( "context" "testing" "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" "github.com/tidbcloud/config-service/sdk/api" "github.com/tidbcloud/config-service/sdk/mock" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" cloudv1 "github.com/tidbcloud/infra-api/api/v1" "github.com/tidbcloud/infra-api/controllers/test" ) func TestOverlayTiDBEndpoint(t *testing.T) { serviceExternalTrafficPolicyTypeLocal := corev1.ServiceExternalTrafficPolicyTypeLocal cases := map[string]struct { cluster *cloudv1.Cluster provider string expectSvc *v1alpha1.TiDBServiceSpec enableGracefulShutdownWebhook bool }{ "AWSEnablePublic": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, }, }, }, }, provider: "aws", enableGracefulShutdownWebhook: false, expectSvc: &v1alpha1.TiDBServiceSpec{ ServiceSpec: v1alpha1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, Annotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": "true", "service.beta.kubernetes.io/aws-load-balancer-internet-facing": "0.0.0.0/0", "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags": "cluster=123,component=nlb-private,tenant=233,usedby=customer-tidb", }, }, ExternalTrafficPolicy: &serviceExternalTrafficPolicyTypeLocal, ExposeStatus: pointer.Bool(true), }, }, "AWSDisablePublic": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(false), }, }, }, }, provider: "aws", enableGracefulShutdownWebhook: false, expectSvc: &v1alpha1.TiDBServiceSpec{ ServiceSpec: v1alpha1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, }, ExposeStatus: pointer.Bool(true), }, }, "GCPEnablePublic": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, }, }, }, }, provider: "gcp", enableGracefulShutdownWebhook: true, expectSvc: &v1alpha1.TiDBServiceSpec{ ServiceSpec: v1alpha1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, Annotations: map[string]string{ "cloud.google.com/load-balancer-type": "External", }, }, AdditionalPorts: []corev1.ServicePort{ { Name: "health-checker", Protocol: corev1.ProtocolTCP, Port: 8080, TargetPort: intstr.FromInt(8080), }, }, ExternalTrafficPolicy: &serviceExternalTrafficPolicyTypeLocal, ExposeStatus: pointer.Bool(true), }, }, "GCPDisablePublic": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(false), }, }, }, }, provider: "gcp", enableGracefulShutdownWebhook: true, expectSvc: &v1alpha1.TiDBServiceSpec{ ServiceSpec: v1alpha1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, }, AdditionalPorts: []corev1.ServicePort{ { Name: "health-checker", Protocol: corev1.ProtocolTCP, Port: 8080, TargetPort: intstr.FromInt(8080), }, }, ExposeStatus: pointer.Bool(true), }, }, "AWSEnablePublicWithSpecificPort": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, Port: pointer.Int32(5000), }, }, }, }, provider: "aws", enableGracefulShutdownWebhook: false, expectSvc: &v1alpha1.TiDBServiceSpec{ ServiceSpec: v1alpha1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, Annotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled": "true", "service.beta.kubernetes.io/aws-load-balancer-internet-facing": "0.0.0.0/0", "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", "service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags": "cluster=123,component=nlb-private,tenant=233,usedby=customer-tidb", }, Port: pointer.Int32(5000), }, ExternalTrafficPolicy: &serviceExternalTrafficPolicyTypeLocal, ExposeStatus: pointer.Bool(true), }, }, "AWSDisablePublicWithSpecificPort": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(false), Port: pointer.Int32(5000), }, }, }, }, provider: "aws", enableGracefulShutdownWebhook: false, expectSvc: &v1alpha1.TiDBServiceSpec{ ServiceSpec: v1alpha1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, Port: pointer.Int32(5000), }, ExposeStatus: pointer.Bool(true), }, }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) r := newFakeReconcileContext( tc.cluster, &ClusterReconciler{ ProviderType: tc.provider, }, nil) r.providerGardenerSpec.EnableGracefulShutdownWebhook = &tc.enableGracefulShutdownWebhook tidbcluster := &v1alpha1.TidbCluster{ Spec: v1alpha1.TidbClusterSpec{ TiDB: &v1alpha1.TiDBSpec{}, }, } g.Expect(r.overlayTiDBEndpoint(tidbcluster)).To(Succeed()) g.Expect(tidbcluster.Spec.TiDB.Service).To(Equal(tc.expectSvc)) }) } } func TestAssembleTidbClusterValues(t *testing.T) { cases := map[string]struct { cluster *cloudv1.Cluster cc *api.DescribeClusterConfigOutput provider string expectYaml string }{ "AWS_NetworkStorage": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Namespace: "test", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "1", "tags.tidbcloud.com/project": "1", "tags.tidbcloud.com/billing-cluster-id": "123", "tags.tidbcloud.com/billing-tenant-id": "1", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"0.0.0.0/0"}, }, }, Cluster: cloudv1.ClusterTopology{ TiDB: cloudv1.TiDB{ Replicas: 2, InstanceType: "compute", }, TiKV: cloudv1.TiKV{ Replicas: 3, InstanceType: "storage", StorageSize: resource.MustParse("1000Gi"), }, PD: cloudv1.PD{ Replicas: 3, InstanceType: "compute", StorageSize: resource.MustParse("10Gi"), }, }, }, }, provider: "aws", cc: getMockedClusterConfig(), expectYaml: `metadata: annotations: billing-tidb-config: '{"tikv":{"provider":"aws","replicas":3,"region":"test-region","instanceType":"storage","disks":[{"diskType":"ssd_network","capactity":1074}]},"tidb":{"provider":"aws","replicas":2,"region":"test-region","instanceType":"compute","disks":null},"tiflash":{"provider":"","replicas":0,"region":"","instanceType":"","disks":null}}' ops.tidbcloud.com/config-commit: TEST ops.tidbcloud.com/config-id: base ops.tidbcloud.com/config-version: "1" tidb.tidb.pingcap.com/skip-tls-when-connect-tidb: "" tidb.pingcap.com/tidb-min-ready-seconds: "180" creationTimestamp: null labels: tenant: "1" project: "1" billing-cluster-id: "123" billing-tenant-id: "1" cloud.pingcap.com/provider-type: "aws" cluster.cloud.pingcap.com/name: "123" cluster.cloud.pingcap.com/namespace: "test" name: db namespace: tidb123 spec: configUpdateStrategy: RollingUpdate discovery: {} enableDynamicConfiguration: true helper: image: gcr.io/pingcap-public/dbaas/busybox:1.31.1 hostNetwork: false imagePullPolicy: IfNotPresent pd: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: pd topologyKey: kubernetes.io/hostname baseImage: gcr.io/pingcap-public/dbaas/pd config: | [dashboard] public-path-prefix = "/proxy/cluster/123/pd/dashboard" [security] cert-allowed-cn = ["13d27bc2"] env: - name: DASHBOARD_SESSION_SECRET valueFrom: secretKeyRef: key: encryption_key name: db-dashboard-session-secret hostNetwork: true mountClusterClientSecret: true nodeSelector: gardener.wg.prefix.name: tidb123-pd replicas: 3 maxFailoverCount: 0 requests: cpu: "7" memory: 12777527705 storage: 10Gi limits: memory: 14388140441 storageClassName: pd-gp3 tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-pd pvReclaimPolicy: Retain schedulerName: default-scheduler tidb: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tidb topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tidb binlogEnabled: false config: | temp-dir = "/data/tmp" [performance] tcp-keep-alive = true [security] cluster-verify-cn = ["13d27bc2"] hostNetwork: true storageVolumes: - name: tidb-add-index-acceleration storageSize: "100Gi" mountPath: "/data/tmp" storageClassName: "tidb-add-index-acceleration-gp3" maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tidb podSecurityContext: sysctls: - name: net.ipv4.tcp_keepalive_intvl value: "75" - name: net.ipv4.tcp_keepalive_time value: "300" replicas: 2 requests: cpu: "7" memory: 12777527705 limits: memory: 14388140441 separateSlowLog: true service: annotations: service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: billing-cluster-id=123,billing-tenant-id=1,cluster=123,component=nlb-private,project=1,tenant=1,usedby=customer-tidb service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" service.beta.kubernetes.io/aws-load-balancer-internet-facing: 0.0.0.0/0 service.beta.kubernetes.io/aws-load-balancer-type: nlb exposeStatus: true externalTrafficPolicy: Local loadBalancerSourceRanges: - 0.0.0.0/0 type: LoadBalancer slowLogTailer: limits: cpu: 100m memory: 50Mi requests: cpu: 20m memory: 5Mi tlsClient: enabled: true tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tidb tikv: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tikv topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tikv config: | [security] cert-allowed-cn = ["13d27bc2"] [security.encryption] data-encryption-method = "aes256-ctr" data-key-rotation-period = "7d" [security.encryption.master-key] key-id = "test-key" region = "test-region" type = "kms" hostNetwork: true maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tikv podSecurityContext: sysctls: - name: net.core.somaxconn value: "32768" privileged: false replicas: 3 requests: cpu: "7" memory: 28925933649 storage: 1000Gi limits: memory: 30536546385 storageClassName: tikv-gp3 tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tikv timezone: UTC tlsCluster: enabled: true version: v9.0.1`, }, "GCP_LocalStorage": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Namespace: "test", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "1", "tags.tidbcloud.com/project": "1", "tags.tidbcloud.com/billing-cluster-id": "123", "tags.tidbcloud.com/billing-tenant-id": "1", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"0.0.0.0/0"}, }, }, Cluster: cloudv1.ClusterTopology{ TiDB: cloudv1.TiDB{ Replicas: 2, InstanceType: "compute", }, TiKV: cloudv1.TiKV{ Replicas: 3, InstanceType: "storage", UseLocalStorage: pointer.Bool(true), StorageSize: resource.MustParse("1000Gi"), }, PD: cloudv1.PD{ Replicas: 3, InstanceType: "compute", StorageSize: resource.MustParse("10Gi"), }, }, }, }, provider: "gcp", cc: getMockedClusterConfig(), expectYaml: `metadata: annotations: billing-tidb-config: '{"tikv":{"provider":"gcp","replicas":3,"region":"test-region","instanceType":"storage","disks":[{"diskType":"ssd_network","capactity":1074}]},"tidb":{"provider":"gcp","replicas":2,"region":"test-region","instanceType":"compute","disks":null},"tiflash":{"provider":"","replicas":0,"region":"","instanceType":"","disks":null}}' ops.tidbcloud.com/config-commit: TEST ops.tidbcloud.com/config-id: base ops.tidbcloud.com/config-version: "1" tidb.tidb.pingcap.com/skip-tls-when-connect-tidb: "" tidb.pingcap.com/tidb-min-ready-seconds: "180" creationTimestamp: null labels: tenant: "1" project: "1" billing-cluster-id: "123" billing-tenant-id: "1" cloud.pingcap.com/provider-type: "gcp" cluster.cloud.pingcap.com/name: "123" cluster.cloud.pingcap.com/namespace: "test" name: db namespace: tidb123 spec: configUpdateStrategy: RollingUpdate discovery: {} enableDynamicConfiguration: true helper: image: gcr.io/pingcap-public/dbaas/busybox:1.31.1 hostNetwork: false imagePullPolicy: IfNotPresent pd: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: pd topologyKey: kubernetes.io/hostname baseImage: gcr.io/pingcap-public/dbaas/pd config: | [dashboard] public-path-prefix = "/proxy/cluster/123/pd/dashboard" [security] cert-allowed-cn = ["13d27bc2"] env: - name: DASHBOARD_SESSION_SECRET valueFrom: secretKeyRef: key: encryption_key name: db-dashboard-session-secret hostNetwork: true mountClusterClientSecret: true nodeSelector: gardener.wg.prefix.name: tidb123-pd replicas: 3 maxFailoverCount: 0 requests: cpu: "7" memory: 12777527705 storage: 10Gi limits: memory: 14388140441 storageClassName: cloud-ssd tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-pd pvReclaimPolicy: Retain schedulerName: default-scheduler tidb: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tidb topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tidb binlogEnabled: false config: | [performance] tcp-keep-alive = true [security] cluster-verify-cn = ["13d27bc2"] hostNetwork: true maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tidb podSecurityContext: sysctls: - name: net.ipv4.tcp_keepalive_intvl value: "75" - name: net.ipv4.tcp_keepalive_time value: "300" replicas: 2 requests: cpu: "7" memory: 12777527705 limits: memory: 14388140441 separateSlowLog: true service: annotations: cloud.google.com/load-balancer-type: External exposeStatus: true externalTrafficPolicy: Local loadBalancerSourceRanges: - 0.0.0.0/0 type: LoadBalancer slowLogTailer: limits: cpu: 100m memory: 50Mi requests: cpu: 20m memory: 5Mi tlsClient: enabled: true tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tidb tikv: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tikv topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tikv config: | [security] cert-allowed-cn = ["13d27bc2"] [security.encryption] data-encryption-method = "aes256-ctr" data-key-rotation-period = "7d" [security.encryption.master-key] type = "kms" key-id = "test-key" region = "test-region" hostNetwork: true maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tikv podSecurityContext: sysctls: - name: net.core.somaxconn value: "32768" env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: name: gcp-kms-credentials key: AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: gcp-kms-credentials key: AWS_SECRET_ACCESS_KEY privileged: false replicas: 3 requests: cpu: "7" memory: 28925933649 storage: 1000Gi limits: memory: 30536546385 storageClassName: local-storage tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tikv timezone: UTC tlsCluster: enabled: true version: v9.0.1`, }, "AWS_HTAP_FROM_CENTRAL": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Namespace: "test", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "1", "tags.tidbcloud.com/project": "1", "tags.tidbcloud.com/cluster": "123", "tags.tidbcloud.com/billing-cluster-id": "123", "tags.tidbcloud.com/billing-tenant-id": "1", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"0.0.0.0/0"}, }, }, Cluster: cloudv1.ClusterTopology{ TiDB: cloudv1.TiDB{ Replicas: 2, InstanceType: "compute", }, TiKV: cloudv1.TiKV{ Replicas: 3, InstanceType: "storage", StorageSize: resource.MustParse("1000Gi"), }, PD: cloudv1.PD{ Replicas: 3, InstanceType: "compute", StorageSize: resource.MustParse("10Gi"), }, TiFlash: cloudv1.TiFlash{ Replicas: 1, InstanceType: "storage", StorageSize: resource.MustParse("1000Gi"), }, }, }, }, provider: "aws", cc: getMockedClusterConfig(), expectYaml: `metadata: annotations: billing-tidb-config: '{"tikv":{"provider":"aws","replicas":3,"region":"test-region","instanceType":"storage","disks":[{"diskType":"ssd_network","capactity":1074}]},"tidb":{"provider":"aws","replicas":2,"region":"test-region","instanceType":"compute","disks":null},"tiflash":{"provider":"aws","replicas":1,"region":"test-region","instanceType":"storage","disks":[{"diskType":"ssd_network","capactity":1074}]}}' ops.tidbcloud.com/config-commit: TEST ops.tidbcloud.com/config-id: base ops.tidbcloud.com/config-version: "1" tidb.tidb.pingcap.com/skip-tls-when-connect-tidb: "" tidb.pingcap.com/tidb-min-ready-seconds: "180" creationTimestamp: null labels: tenant: "1" project: "1" cluster: "123" billing-cluster-id: "123" billing-tenant-id: "1" cloud.pingcap.com/provider-type: "aws" cluster.cloud.pingcap.com/name: "123" cluster.cloud.pingcap.com/namespace: "test" name: db namespace: tidb123 spec: configUpdateStrategy: RollingUpdate discovery: {} enableDynamicConfiguration: true helper: image: gcr.io/pingcap-public/dbaas/busybox:1.31.1 hostNetwork: false imagePullPolicy: IfNotPresent pd: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: pd topologyKey: kubernetes.io/hostname baseImage: gcr.io/pingcap-public/dbaas/pd config: | [dashboard] public-path-prefix = "/proxy/cluster/123/pd/dashboard" [replication] enable-placement-rules = true [security] cert-allowed-cn = ["13d27bc2"] env: - name: DASHBOARD_SESSION_SECRET valueFrom: secretKeyRef: key: encryption_key name: db-dashboard-session-secret hostNetwork: true mountClusterClientSecret: true nodeSelector: gardener.wg.prefix.name: tidb123-pd replicas: 3 maxFailoverCount: 0 requests: cpu: "7" memory: 12777527705 storage: 10Gi limits: memory: 14388140441 storageClassName: pd-gp3 tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-pd pvReclaimPolicy: Retain schedulerName: default-scheduler tidb: additionalContainers: additionalVolumes: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tidb topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tidb binlogEnabled: false config: | temp-dir = "/data/tmp" [performance] tcp-keep-alive = true [security] cluster-verify-cn = ["13d27bc2"] hostNetwork: true storageVolumes: - name: tidb-add-index-acceleration storageSize: "100Gi" mountPath: "/data/tmp" storageClassName: "tidb-add-index-acceleration-gp3" maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tidb podSecurityContext: sysctls: - name: net.ipv4.tcp_keepalive_intvl value: "75" - name: net.ipv4.tcp_keepalive_time value: "300" replicas: 2 requests: cpu: "7" memory: 12777527705 limits: memory: 14388140441 separateSlowLog: true service: annotations: service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: billing-cluster-id=123,billing-tenant-id=1,cluster=123,component=nlb-private,project=1,tenant=1,usedby=customer-tidb service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" service.beta.kubernetes.io/aws-load-balancer-internet-facing: 0.0.0.0/0 service.beta.kubernetes.io/aws-load-balancer-type: nlb exposeStatus: true externalTrafficPolicy: Local loadBalancerSourceRanges: - 0.0.0.0/0 type: LoadBalancer slowLogTailer: limits: cpu: 100m memory: 50Mi requests: cpu: 20m memory: 5Mi tlsClient: enabled: true tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tidb tiflash: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tiflash topologyKey: kubernetes.io/hostname baseImage: gcr.io/pingcap-public/dbaas/tiflash config: config: | [security] cert_allowed_cn = ["13d27bc2"] proxy: | [security] cert-allowed-cn = ["13d27bc2"] redact-info-log = true [security.encryption] data-encryption-method = "aes256-ctr" data-key-rotation-period = "7d" [security.encryption.master-key] key-id = "test-key" region = "test-region" type = "kms" hostNetwork: true logTailer: limits: cpu: 100m memory: 50Mi requests: cpu: 20m memory: 5Mi nodeSelector: gardener.wg.prefix.name: tidb123-tiflash podSecurityContext: {} replicas: 1 maxFailoverCount: 0 requests: cpu: "7" memory: 28925933649 storage: 1000Gi limits: memory: 30536546385 storageClaims: - resources: requests: storage: 1000Gi storageClassName: tiflash-gp3 tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tiflash tikv: additionalContainers: additionalVolumes: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tikv topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tikv config: | [security] cert-allowed-cn = ["13d27bc2"] [security.encryption] data-encryption-method = "aes256-ctr" data-key-rotation-period = "7d" [security.encryption.master-key] key-id = "test-key" region = "test-region" type = "kms" hostNetwork: true maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tikv podSecurityContext: sysctls: - name: net.core.somaxconn value: "32768" privileged: false replicas: 3 requests: cpu: "7" memory: 28925933649 storage: 1000Gi limits: memory: 30536546385 storageClassName: tikv-gp3 tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tikv timezone: UTC tlsCluster: enabled: true version: v9.0.1`, }, "GCP_TinyTier": { cluster: &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Namespace: "test", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "1", "tags.tidbcloud.com/project": "1", "tags.tidbcloud.com/billing-cluster-id": "123", "tags.tidbcloud.com/billing-tenant-id": "1", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"0.0.0.0/0"}, }, }, Cluster: cloudv1.ClusterTopology{ TiDB: cloudv1.TiDB{ Replicas: 2, InstanceType: "tiny", }, TiKV: cloudv1.TiKV{ Replicas: 3, InstanceType: "tiny", StorageSize: resource.MustParse("100Gi"), }, PD: cloudv1.PD{ Replicas: 3, InstanceType: "tiny", StorageSize: resource.MustParse("10Gi"), }, }, }, }, provider: "gcp", cc: getMockedClusterConfig(), expectYaml: `metadata: annotations: billing-tidb-config: '{"tikv":{"provider":"gcp","replicas":3,"region":"test-region","instanceType":"tiny","disks":[{"diskType":"ssd_network","capactity":108}]},"tidb":{"provider":"gcp","replicas":2,"region":"test-region","instanceType":"tiny","disks":null},"tiflash":{"provider":"","replicas":0,"region":"","instanceType":"","disks":null}}' ops.tidbcloud.com/config-commit: TEST ops.tidbcloud.com/config-id: base ops.tidbcloud.com/config-version: "1" tidb.tidb.pingcap.com/skip-tls-when-connect-tidb: "" tidb.pingcap.com/tidb-min-ready-seconds: "180" creationTimestamp: null labels: tenant: "1" project: "1" billing-cluster-id: "123" billing-tenant-id: "1" cloud.pingcap.com/provider-type: "gcp" cluster.cloud.pingcap.com/name: "123" cluster.cloud.pingcap.com/namespace: "test" name: db namespace: tidb123 spec: configUpdateStrategy: RollingUpdate discovery: {} enableDynamicConfiguration: true helper: image: gcr.io/pingcap-public/dbaas/busybox:1.31.1 hostNetwork: false imagePullPolicy: IfNotPresent pd: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: pd topologyKey: kubernetes.io/hostname baseImage: gcr.io/pingcap-public/dbaas/pd config: | [dashboard] public-path-prefix = "/proxy/cluster/123/pd/dashboard" [security] cert-allowed-cn = ["13d27bc2"] env: - name: DASHBOARD_SESSION_SECRET valueFrom: secretKeyRef: key: encryption_key name: db-dashboard-session-secret hostNetwork: true mountClusterClientSecret: true nodeSelector: gardener.wg.prefix.name: tidb123-pd replicas: 3 maxFailoverCount: 0 requests: cpu: "0" memory: 0 storage: 10Gi limits: memory: 1610612736 storageClassName: cloud-ssd tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-pd pvReclaimPolicy: Retain schedulerName: default-scheduler tidb: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tidb topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tidb binlogEnabled: false config: | [performance] tcp-keep-alive = true [security] cluster-verify-cn = ["13d27bc2"] hostNetwork: true maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tidb podSecurityContext: sysctls: - name: net.ipv4.tcp_keepalive_intvl value: "75" - name: net.ipv4.tcp_keepalive_time value: "300" replicas: 2 requests: cpu: "0" memory: 0 limits: memory: 1610612736 separateSlowLog: true service: annotations: cloud.google.com/load-balancer-type: External exposeStatus: true externalTrafficPolicy: Local loadBalancerSourceRanges: - 0.0.0.0/0 type: LoadBalancer slowLogTailer: limits: cpu: 100m memory: 50Mi requests: cpu: 20m memory: 5Mi tlsClient: enabled: true tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tidb tikv: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchLabels: app.kubernetes.io/component: tikv topologyKey: kubernetes.io/hostname annotations: tidb.pingcap.com/sysctl-init: "true" baseImage: gcr.io/pingcap-public/dbaas/tikv config: | [security] cert-allowed-cn = ["13d27bc2"] [security.encryption] data-encryption-method = "aes256-ctr" data-key-rotation-period = "7d" [security.encryption.master-key] type = "kms" key-id = "test-key" region = "test-region" hostNetwork: true maxFailoverCount: 0 nodeSelector: gardener.wg.prefix.name: tidb123-tikv podSecurityContext: sysctls: - name: net.core.somaxconn value: "32768" env: - name: AWS_ACCESS_KEY_ID valueFrom: secretKeyRef: name: gcp-kms-credentials key: AWS_ACCESS_KEY_ID - name: AWS_SECRET_ACCESS_KEY valueFrom: secretKeyRef: name: gcp-kms-credentials key: AWS_SECRET_ACCESS_KEY privileged: false replicas: 3 requests: cpu: "0" memory: 0 storage: 100Gi limits: memory: 1610612736 storageClassName: cloud-ssd tolerations: - effect: NoSchedule key: dedicated operator: Equal value: tidb-tikv timezone: UTC tlsCluster: enabled: true version: v9.0.1`, }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) configService := &mock.MockClient{ ClusterConfigOutput: tc.cc, } r := newFakeReconcileContext( tc.cluster, &ClusterReconciler{ ProviderType: tc.provider, ConfigService: configService, }, nil) pdMixedDeployConfig := &cloudv1.MixedDeployPDWithTiKV{} pdMixedDeployConfig.SetAlwaysEnabled() result, err := r.AssembleTidbClusterValues(pdMixedDeployConfig) // test marshal then unmarshal the assembled tidbcluster value g.Expect(err).NotTo(HaveOccurred()) resultMarshaled, err := yaml.Marshal(result) g.Expect(err).NotTo(HaveOccurred()) var resultUnmarshaled v1alpha1.TidbCluster err = yaml.Unmarshal(resultMarshaled, &resultUnmarshaled) g.Expect(err).NotTo(HaveOccurred()) var expect v1alpha1.TidbCluster err = yaml.Unmarshal([]byte(tc.expectYaml), &expect) g.Expect(err).NotTo(HaveOccurred()) diff := cmp.Diff(&expect, &resultUnmarshaled) if diff != "" { t.Errorf("unexpected (-want, +got): %s", diff) } }) } } func getMockedClusterConfig() *api.DescribeClusterConfigOutput { return &api.DescribeClusterConfigOutput{ // [Deprecated on 2022-Jun-25] Set a version that TopSQL is supported. Version: "v9.0.1", ConfigMeta: &api.ConfigMeta{ ID: "base", GitCommitID: "TEST", ConfigVersion: 1, }, Components: []api.ComponentOutput{{ Component: api.ComponentPD, Image: api.Image{BaseImage: "gcr.io/pingcap-public/dbaas/pd"}, Config: api.NewTomlConfig(map[string]interface{}{}), }, { Component: api.ComponentTiDB, Image: api.Image{BaseImage: "gcr.io/pingcap-public/dbaas/tidb"}, Config: api.NewTomlConfig(map[string]interface{}{}), }, { Component: api.ComponentTiKV, Image: api.Image{BaseImage: "gcr.io/pingcap-public/dbaas/tikv"}, Config: api.NewTomlConfig(map[string]interface{}{}), }, { Component: api.ComponentTiFlash, Image: api.Image{BaseImage: "gcr.io/pingcap-public/dbaas/tiflash"}, Config: api.NewTomlConfig(map[string]interface{}{}), }, { Component: api.ComponentTiFlashLearner, Image: api.Image{BaseImage: "gcr.io/pingcap-public/dbaas/tiflash"}, Config: api.NewTomlConfig(map[string]interface{}{ "security": map[string]interface{}{ "redact-info-log": true, }, }), }}, } } func TestEnableSecurityEnhancedMode(t *testing.T) { cluster := &cloudv1.Cluster{ ObjectMeta: metav1.ObjectMeta{ Name: "123", Labels: map[string]string{ "tags.tidbcloud.com/tenant": "233", }, }, Spec: cloudv1.ClusterSpec{ SqlEndpoints: cloudv1.SqlEndpoints{ Public: cloudv1.Service{ Enabled: pointer.Bool(true), LoadBalancerSourceRanges: []string{"128.128.128.128/32"}, }, }, }, } shootClient := &test.MockClient{ MockUpdate: func(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error { return nil }, MockGet: func(_ context.Context, _ client.ObjectKey, obj runtime.Object) error { if init, ok := obj.(*v1alpha1.TidbInitializer); ok { init.Status = v1alpha1.TidbInitializerStatus{ Phase: v1alpha1.InitializePhaseCompleted, JobStatus: batchv1.JobStatus{ Succeeded: 1, }, } } return nil }, } mkCluster := func() *v1alpha1.TidbCluster { return &v1alpha1.TidbCluster{ Spec: v1alpha1.TidbClusterSpec{ Version: "v5.2", TiDB: &v1alpha1.TiDBSpec{ Config: v1alpha1.NewTiDBConfig(), }, }, } } t.Run("SEM", func(t *testing.T) { g := NewGomegaWithT(t) r := newFakeReconcileContext(cluster, &ClusterReconciler{ProviderType: "aws"}, shootClient) { r.tc = mkCluster() r.tc.Spec.Version = "v5.2" g.Expect(r.ensureSecurityEnhancedMode(r.ctx)).To(Succeed()) v := r.tc.Spec.TiDB.Config.Get("security.enable-sem") g.Expect(v).To(BeNil()) } { r.tc = mkCluster() r.tc.Spec.Version = "v5.3" g.Expect(r.ensureSecurityEnhancedMode(r.ctx)).To(Succeed()) v := r.tc.Spec.TiDB.Config.Get("security.enable-sem") g.Expect(v).To(Not(BeNil())) } }) } func TestConvertSuspendActions(t *testing.T) { cases := map[string]struct { action *cloudv1.SuspendAction expectAction *v1alpha1.SuspendAction }{ "nil action": { action: nil, expectAction: nil, }, "true": { action: &cloudv1.SuspendAction{ SuspendCompute: true, }, expectAction: &v1alpha1.SuspendAction{ SuspendStatefulSet: true, }, }, "false": { action: &cloudv1.SuspendAction{ SuspendCompute: false, }, expectAction: &v1alpha1.SuspendAction{ SuspendStatefulSet: false, }, }, "empty": { action: &cloudv1.SuspendAction{}, expectAction: &v1alpha1.SuspendAction{ SuspendStatefulSet: false, }, }, } for name, tc := range cases { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) result := convertSuspendActions(tc.action) if tc.action != nil { g.Expect((*result).SuspendStatefulSet).Should(Equal((*tc.expectAction).SuspendStatefulSet)) } else { g.Expect(result).Should(Equal(tc.expectAction)) } }) } }