package imports import ( "fmt" "testing" "github.com/google/go-cmp/cmp" . "github.com/onsi/gomega" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" cloudv1 "github.com/tidbcloud/infra-api/api/v1" "github.com/tidbcloud/infra-api/pkg/consts" "github.com/tidbcloud/infra-provider/util/config" ) func TestLightningService(t *testing.T) { var ( fakeName = "fakeName" fakeNS = "fakeNamespace" instance = "db" ) cases := map[string]struct { expectSpec string }{ "Import_Service": {` ports: - port: 8289 protocol: TCP targetPort: 8289 selector: app.kubernetes.io/name: fakeName`}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) svc := newLightningService(fakeName, fakeNS, instance) var expectSvc corev1.Service expectYaml := fmt.Sprintf(`metadata: creationTimestamp: null labels: app.kubernetes.io/component: tidb-lightning app.kubernetes.io/instance: db app.kubernetes.io/name: fakeName name: fakeName namespace: fakeNamespace spec:%s`, tc.expectSpec) err := yaml.Unmarshal([]byte(expectYaml), &expectSvc) g.Expect(err).NotTo(HaveOccurred()) diff := cmp.Diff(&expectSvc, svc) g.Expect(diff).To(BeEmpty()) }) } } func TestPersistentVolumeClaim(t *testing.T) { var ( fakeName = "fakeName" fakeNS = "fakeNamespace" fakeSC = "fakeStorageClass" fakeRL = "100" instance = "db" ) cases := map[string]struct { expectSpec string }{ "Import_Service": {` accessModes: - ReadWriteOnce resources: requests: storage: "100" storageClassName: fakeStorageClass`}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) pvc := newPersistentVolumeClaim(fakeName, fakeNS, instance, fakeSC, fakeRL) var expectPvc corev1.PersistentVolumeClaim expectYaml := fmt.Sprintf(`metadata: creationTimestamp: null labels: app.kubernetes.io/component: tidb-lightning app.kubernetes.io/instance: db app.kubernetes.io/name: fakeName name: fakeName namespace: fakeNamespace spec:%s`, tc.expectSpec) err := yaml.Unmarshal([]byte(expectYaml), &expectPvc) g.Expect(err).NotTo(HaveOccurred()) diff := cmp.Diff(&expectPvc, pvc) g.Expect(diff).To(BeEmpty()) }) } } func TestLightningSecret(t *testing.T) { var ( fakeName = "fakeName" fakeNS = "fakeNamespace" fakeCfg = "fakeConfig" fakeCred = []byte("fakeCredentials") instance = "db" ) cases := map[string]struct { provider string cred []byte expectData string }{ "aws-cred-nonzero": {consts.ProviderTypeAWS, fakeCred, ` credentiascriptls: ZmFrZUNyZWRlbnRpYWxz tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "aws-cred-zero": {consts.ProviderTypeAWS, nil, ` tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "aws-free-tier-cred-nonzero": {consts.ProviderTypeAWSFreeTier, fakeCred, ` credentiascriptls: ZmFrZUNyZWRlbnRpYWxz tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "aws-free-tier-cred-zero": {consts.ProviderTypeAWSFreeTier, nil, ` tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "aws-shared-tier-cred-nonzero": {consts.ProviderTypeAWSSharedTier, fakeCred, ` credentiascriptls: ZmFrZUNyZWRlbnRpYWxz tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "aws-shared-tier-cred-zero": {consts.ProviderTypeAWSSharedTier, nil, ` tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "gcp-cred-nonzero": {consts.ProviderTypeGCP, fakeCred, ` privateKey: ZmFrZUNyZWRlbnRpYWxz tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "gcp-cred-zero": {consts.ProviderTypeGCP, nil, ` privateKey: null tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "gcp-shared-tier-cred-nonzero": {consts.ProviderTypeGCP, fakeCred, ` privateKey: ZmFrZUNyZWRlbnRpYWxz tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, "gcp-shared-tier-cred-zero": {consts.ProviderTypeGCP, nil, ` privateKey: null tidb-lightning.toml: ZmFrZUNvbmZpZw==`}, } for name, tc := range cases { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) sec := newLightningSecret(tc.provider, fakeName, fakeNS, instance, fakeCfg, tc.cred) var expectSec corev1.Secret expectYaml := fmt.Sprintf(` metadata: creationTimestamp: null labels: app.kubernetes.io/component: tidb-lightning app.kubernetes.io/instance: db app.kubernetes.io/name: fakeName name: fakeName namespace: fakeNamespace data:%s`, tc.expectData) err := yaml.Unmarshal([]byte(expectYaml), &expectSec) g.Expect(err).NotTo(HaveOccurred()) diff := cmp.Diff(&expectSec, sec) g.Expect(diff).To(BeEmpty()) }) } } func TestImportErrorMessageDigest(t *testing.T) { g := NewGomegaWithT(t) testCases := []struct { message string expected string }{ { message: "awfeasdfwaer\nasdfqwae\nfwe", expected: "awfeasdfwaer", }, { message: "RequestCanceled: request context canceled\ncaused by: context canceled\ngithub.com/ping", expected: "RequestCanceled: request context canceled", }, { message: "RequestCanceled: request context canceled", expected: "RequestCanceled: request context canceled", }, { message: "\n", expected: "", }, { message: "", expected: "", }, { message: multiErr, expected: "[tikv:9005]Region is unavailable", }, } for _, tc := range testCases { g.Expect(tc.expected).To(Equal(importErrorMessageDigest(tc.message))) } } var multiErr = "the following errors occurred:\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598\n - [tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598" func TestUnwrapMultiErr(t *testing.T) { testCases := []struct { message string expected string }{ { message: multiErr, expected: "[tikv:9005]Region is unavailable\n github.com/pingcap/errors.AddStack\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/errors.go:174\n github.com/pingcap/errors.Trace\n \t/go/pkg/mod/github.com/pingcap/errors@v0.11.5-0.20221009092201-b66cddb77c32/juju_adaptor.go:15\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleCopResponse\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1322\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTaskOnce\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1222\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).handleTask\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:1105\n github.com/pingcap/tidb/store/copr.(*copIteratorWorker).run\n \t/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/br/store/copr/coprocessor.go:798\n runtime.goexit\n \t/usr/local/go/src/runtime/asm_amd64.s:1598", }, } for _, tc := range testCases { require.Equal(t, tc.expected, unwrapMultiErr(tc.message)) } } func TestAdjustLightningConfig(t *testing.T) { testCases := []struct { name string config string baseConfig ProviderBaseConfig expected string }{ { name: "default", config: `[lightning] `, baseConfig: ProviderBaseConfig{ DataFormat: cloudv1.DataFormatRawParquet, CpuCore: 32, }, expected: `[lightning] region-concurrency = 22 `, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { cfg := config.New(map[string]interface{}{}) err := cfg.UnmarshalTOML([]byte(tc.config)) require.NoError(t, err) AdjustLightningConfig(cfg, tc.baseConfig) expectCfg := config.New(map[string]interface{}{}) err = expectCfg.UnmarshalTOML([]byte(tc.expected)) require.NoError(t, err) require.Equal(t, expectCfg, cfg) }) } }