package tidbcluster

import (
	"context"
	"database/sql"
	"fmt"
	"net/http"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	pkgErrors "github.com/pkg/errors"
	prometheus "github.com/prometheus/client_golang/api"
	prometheus_api "github.com/prometheus/client_golang/api/prometheus/v1"
	"github.com/prometheus/common/model"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	apiresource "github.com/pingcap/test-infra/common/model/resource"
	"github.com/pingcap/test-infra/sdk/e2etest/framework"
	logsearch "github.com/pingcap/test-infra/sdk/pkg/logs"
	sdkresource "github.com/pingcap/test-infra/sdk/resource"
	"github.com/pingcap/test-infra/sdk/resource/impl/k8s"
)

var _ = Describe("TiDBCluster Test", func() {
	Context("TiDBCluster.Create", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-create")

		It("TiDBCluster's testbed should equal to testbed's name", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			Expect(tc.Testbed).Should(Equal(f.Testbed()))
			Expect(tc.Spec.PD.Replicas).Should(Equal(1))
			Expect(tc.Provider).Should(Equal(sdkresource.FQDN(k8s.TiDBClusterClient{})))

			spec, err := tc.GetSpec(sdkresource.TiDB)
			framework.ExpectNoError(err)
			toml, err := spec.Config.MarshalTOML()
			framework.ExpectNoError(err)
			Expect(string(toml)).Should(Equal(`[log]
  level = "info"
  not-exist-config = 1
  slow-query-file = "/var/lib/tidb/log/slowquery"
  [log.file]
    filename = "/var/lib/tidb/log/tidb.log"
    max-backups = 3
`))

			spec, err = tc.GetSpec(sdkresource.TiKV)
			framework.ExpectNoError(err)
			toml, err = spec.Config.MarshalTOML()
			framework.ExpectNoError(err)
			Expect(string(toml)).Should(Equal(`log-file = "/var/lib/tikv/log/tikv.log"

[rocksdb]
  max-open-files = 256

[storage]
  reserve-space = "100MB"
`))

			spec, err = tc.GetSpec(sdkresource.PD)
			framework.ExpectNoError(err)
			toml, err = spec.Config.MarshalTOML()
			framework.ExpectNoError(err)
			Expect(string(toml)).Should(Equal(`[log]
  [log.file]
    filename = "/var/lib/pd/log/pd.log"
`))

		})

		It("TiDBCluster deploys with tiflash", func() {
			tc := f.CreateTiDBClusterWithTiflash(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			Expect(tc.Spec.TiFlash.Replicas).Should(Equal(1))

			spec, err := tc.GetSpec(sdkresource.TiFlash)
			framework.ExpectNoError(err)
			toml, err := spec.Config.MarshalTOML()
			framework.ExpectNoError(err)
			Expect(string(toml)).Should(Equal(`config = "log-level = \"info\"\n"
`))
		})

		It("TiDBCluster creates with maxReplicas annotation", func() {
			tc := f.CreateTiDBClusterWithMaxReplicas(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			pod, err := f.KubeClientSetPack.ClientSet.
				CoreV1().
				Pods(tc.Namespace()).
				Get(context.TODO(), fmt.Sprintf("%s-tidb-0", tc.Name()), metav1.GetOptions{})
			framework.ExpectNoError(err)
			Expect(pod.Annotations["test-infra.pingcap.net/max-replicas"]).Should(Equal("2"))
		})

		It("TiDBCluster creates with postStart sql script", func() {
			tc := f.CreateTiDBClusterWithMPostStartScript(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			addr, err := tc.ServiceURL(sdkresource.DBAddr)
			framework.ExpectNoError(err)

			dsn := sdkresource.BuildDSN(addr, "test_infra_e2e")
			db, err := sql.Open("mysql", dsn)
			if err != nil {
				framework.ExpectNoError(err)
			}
			defer db.Close()
			_, err = db.Exec("insert into t values(1)")
			framework.ExpectNoError(err)
		})

		// need to run bin/tc-validator
		PIt("TiDBCluster's create works well when open validator", func() {
			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
					SpecValidator: &k8s.TiDBClusterSpecValidator{
						TcValidatorURL: "http://127.0.0.1:8080/api/v1/validate",
					},
				},
			})
			f.CreateTiDBCluster(f.CreateTestbed())
		})

		// need to run bin/tc-validator
		PIt("TiDBCluster's create works well when open validator on error spec", func() {
			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
					SpecValidator: &k8s.TiDBClusterSpecValidator{
						TcValidatorURL: "http://127.0.0.1:8080/api/v1/validate",
					},
				},
			})
			err := f.CreateTiDBClusterWithError(f.CreateTestbed())
			framework.Logf("err: %s", err)
			Expect(strings.TrimSpace(err.Error())).Should(Equal(`validator tidbcluster spec: validator error: {"message":"code=400, message=quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$', internal=quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'","reason":"BadRequest","code":400}`))
		})
	})

	Context("TiDBCluster.UpdateConfig", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-update")

		It("TiDBCluster should be updated", func() {
			var cfgMP apiresource.GenericConfig
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)
			err := f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiKV,
				Config:  "log-level = \"debug\"",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ := tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiKV,
				Command: "cat /etc/tikv/tikv.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			framework.ExpectNoError(err)
			Expect(cfgMP.MP["log-level"].(string)).Should(Equal("debug"))
			Expect(cfgMP.MP["rocksdb"].(map[string]interface{})["max-open-files"].(int64)).Should(Equal(int64(256)))

			f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiKV,
				Config:  "[rocksdb]\nmax-open-files = 128\n[storage]\nreserve-space = \"150MB\"",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ = tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiKV,
				Command: "cat /etc/tikv/tikv.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			framework.ExpectNoError(err)
			Expect(cfgMP.MP["storage"].(map[string]interface{})["reserve-space"].(string)).Should(Equal("150MB"))
			Expect(cfgMP.MP["rocksdb"].(map[string]interface{})["max-open-files"].(int64)).Should(Equal(int64(128)))

			err = f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.PD,
				Config:  "[log]\nlevel = \"debug\"",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ = tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.PD,
				Command: "cat /etc/pd/pd.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			framework.ExpectNoError(err)
			Expect(cfgMP.MP["log"].(map[string]interface{})["level"].(string)).Should(Equal("debug"))

			err = f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiDB,
				Config:  "[log]\nlevel = \"debug\"",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ = tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiDB,
				Command: "cat /etc/tidb/tidb.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			framework.ExpectNoError(err)
			Expect(cfgMP.MP["log"].(map[string]interface{})["level"].(string)).Should(Equal("debug"))
		})

		It("TiDBCluster many components should be updated", func() {
			var cfgMP apiresource.GenericConfig
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			err := f.UpdateTiDBClusterConfigs(tc, sdkresource.TiDBClusterUpdateConfigsOptions{Configs: map[sdkresource.TiDBCompType]string{
				sdkresource.TiDB: "[log]\nlevel = \"debug\"",
				sdkresource.PD:   "[log]\nlevel = \"debug\"",
			}, Timeout: 5 * time.Minute})
			framework.ExpectNoError(err)
			out, _, _, _ := tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiDB,
				Command: "cat /etc/tidb/tidb.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			framework.ExpectNoError(err)
			Expect(cfgMP.MP["log"].(map[string]interface{})["level"].(string)).Should(Equal("debug"))
			out, _, _, _ = tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.PD,
				Command: "cat /etc/pd/pd.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			framework.ExpectNoError(err)
			Expect(cfgMP.MP["log"].(map[string]interface{})["level"].(string)).Should(Equal("debug"))
		})

		It("TiDBCluster with ticdc should be updated", func() {
			tc := f.CreateTiDBClusterWithTiCDC(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)
			err := f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiCDC,
				Config:  "log-level = \"debug\"",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ := tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiCDC,
				Command: "ps",
				Peer:    0,
			})
			framework.ExpectNoError(err)
			Expect(strings.Contains(out, "--log-level=debug")).Should(Equal(true))
		})

		It("TiDBCluster with tiflash should be updated", func() {
			var cfgMP apiresource.GenericConfig
			tc := f.CreateTiDBClusterWithTiflash(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)
			err := f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiFlash,
				Config:  "log-level = \"debug\"",
				Timeout: 5 * time.Minute,
			})
			framework.ExpectNoError(err)
			out, _, _, _ := tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiFlash,
				Command: "cat /data0/config.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))
			Expect(cfgMP.MP["log-level"].(string)).Should(Equal("debug"))
		})

		It("TiDBCluster update with none updated config", func() {
			var cfgMP apiresource.GenericConfig
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			err := f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiDB,
				Config:  "",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ := tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiDB,
				Command: "cat /etc/tidb/tidb.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))

			framework.ExpectError(err)
			Expect(err.Error()).Should(Equal("no updated config"))
			Expect(cfgMP.MP["log"].(map[string]interface{})["level"].(string)).Should(Equal("info"))

			err = f.UpdateTiDBClusterConfig(tc, sdkresource.TiDBClusterUpdateConfigOptions{
				Target:  sdkresource.TiDB,
				Config:  "[log]\nlevel = \"info\"",
				Timeout: 5 * time.Minute,
			})
			out, _, _, _ = tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiDB,
				Command: "cat /etc/tidb/tidb.toml",
				Peer:    0,
			})
			_ = cfgMP.UnmarshalTOML([]byte(out))

			framework.ExpectError(err)
			Expect(err.Error()).Should(Equal("no updated config"))
			Expect(cfgMP.MP["log"].(map[string]interface{})["level"].(string)).Should(Equal("info"))
		})
	})

	Context("TiDBCluster.Scale", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-scale")

		It("TiDBCluster scale in/out should work well", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)
			tidbStatus, err := tc.GetStatus(sdkresource.TiDB)
			framework.ExpectNoError(err)
			framework.ExpectNoError(tc.Scale(sdkresource.TiDBClusterScaleOptions{
				Target:   sdkresource.TiDB,
				Replicas: uint32(tidbStatus.Status.Replicas + 1),
				Timeout:  0,
			}))
			status, err := tc.GetStatus(sdkresource.TiDB)
			framework.ExpectNoError(err)
			Expect(status.Status.Replicas).Should(Equal(tidbStatus.Status.Replicas + 1))

			framework.ExpectNoError(tc.Scale(sdkresource.TiDBClusterScaleOptions{
				Target:   sdkresource.TiDB,
				Replicas: uint32(status.Status.Replicas - 1),
				Timeout:  0,
			}))
			tidbStatus, err = tc.GetStatus(sdkresource.TiDB)
			framework.ExpectNoError(err)
			Expect(status.Status.Replicas - 1).Should(Equal(tidbStatus.Status.Replicas))
		})

		It("TiDBCluster scale in a ticdc instance should work well", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)
			ticdcStatus, err := tc.GetStatus(sdkresource.TiCDC)
			framework.ExpectNoError(err)
			framework.ExpectNoError(tc.Scale(sdkresource.TiDBClusterScaleOptions{
				Target:   sdkresource.TiCDC,
				Replicas: uint32(ticdcStatus.Status.Replicas + 1),
				Timeout:  0,
			}))
			status, err := tc.GetStatus(sdkresource.TiCDC)
			framework.ExpectNoError(err)
			Expect(status.Status.Replicas).Should(Equal(ticdcStatus.Status.Replicas + 1))

			framework.ExpectNoError(tc.Scale(sdkresource.TiDBClusterScaleOptions{
				Target:   sdkresource.TiCDC,
				Replicas: uint32(status.Status.Replicas - 1),
				Timeout:  0,
			}))
			ticdcStatus, err = tc.GetStatus(sdkresource.TiCDC)
			framework.ExpectNoError(err)
			Expect(status.Status.Replicas - 1).Should(Equal(ticdcStatus.Status.Replicas))
		})
	})

	Context("TiDBCluster.Get", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-get")

		It("TiDBCluster get should work well", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			Expect(tc.Testbed).Should(Equal(f.Testbed()))
			Expect(tc.Spec.PD.Replicas).Should(Equal(1))
			Expect(tc.Provider).Should(Equal(sdkresource.FQDN(k8s.TiDBClusterClient{})))
		})

		It("TiDBCluster get should works as well after cluster is ready", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			Expect(tc.Testbed).Should(Equal(f.Testbed()))
			Expect(*tc.Ready).Should(Equal(true))
		})
	})

	Context("TiDBCluster.GetEndpoints", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-getendpoints")

		It("TiDBCluster.GetEndpoints should return with DSN after ready", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			endpoints := f.GetTiDBClusterEndpoints(tc.Handle())

			Expect(len(endpoints)).Should(Equal(4))

			var (
				endpointDsn        *apiresource.Endpoint
				endpointGrafana    *apiresource.Endpoint
				endpointPrometheus *apiresource.Endpoint
				endpointOther      *apiresource.Endpoint
			)
			for i, _ := range endpoints {
				switch endpoints[i].Type {
				case apiresource.EndpointTypeDsn:
					endpointDsn = &endpoints[i]
				case apiresource.EndpointTypeGrafana:
					endpointGrafana = &endpoints[i]
				case apiresource.EndpointTypePrometheus:
					endpointPrometheus = &endpoints[i]
				case apiresource.EndpointTypeOther:
					endpointOther = &endpoints[i]
				}
			}

			Expect(endpointDsn).ShouldNot(Equal(nil))
			db, err := sql.Open("mysql", endpointDsn.ExternalSvcAddress)
			framework.ExpectNoError(err)
			framework.ExpectNoError(db.PingContext(context.TODO()))
			defer db.Close()

			Expect(endpointGrafana).ShouldNot(Equal(nil))
			_, err = http.Get(endpointGrafana.ExternalSvcAddress)
			framework.ExpectNoError(err)

			Expect(endpointPrometheus).ShouldNot(Equal(nil))
			_, err = http.Get(endpointPrometheus.ExternalSvcAddress)
			framework.ExpectNoError(err)

			Expect(endpointOther).ShouldNot(Equal(nil))
			_, err = http.Get(endpointOther.ExternalSvcAddress)
			framework.ExpectNoError(err)
		})
	})

	Context("TiDBCluster.Delete", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-delete")

		It("TiDBCluster delete should works", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			f.DeleteTiDBClusterWaited(tc)

			pods := f.AllPods()
			Expect(len(pods.Items)).Should(Equal(0))
			Expect(len(f.AllPVCs().Items)).Should(Equal(0))
			Expect(len(f.AllServices().Items)).Should(Equal(0))
			Expect(len(f.AllConfigmaps().Items)).Should(Equal(1))
		})
	})

	Context("TiDBCluster.Upgrade component", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-upgrade")

		It("TiDBCluster upgrade should work with ticdc", func() {

			tc := f.CreateTiDBClusterWithTiCDC(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})

			err := tc.Upgrade(sdkresource.TiDBClusterUpgradeOptions{
				ComponentUpgradeOption: &sdkresource.TiDBClusterComponentUpgradeOption{
					Target:  sdkresource.TiCDC,
					Version: "v5.0.1",
				},
				Timeout: 5 * time.Minute,
			})
			framework.ExpectNoError(err)
		})

		It("TiDBCluster upgrade timeout should work with ticdc", func() {

			tc := f.CreateTiDBClusterWithTiCDC(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})

			err := tc.Upgrade(sdkresource.TiDBClusterUpgradeOptions{
				ComponentUpgradeOption: &sdkresource.TiDBClusterComponentUpgradeOption{
					Target:  sdkresource.TiCDC,
					Version: "v5.0.1",
				},
				Timeout: 1 * time.Second,
			})
			Expect(pkgErrors.Cause(err)).Should(Equal(k8s.ErrUpgradeTimeout))
		})
	})

	Context("TiDBCluster.Upgrade cluster", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-upgrade-cluster")

		It("TiDBCluster upgrade cluster should work", func() {
			tc := f.CreateTiDBClusterWithTiCDC(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})

			err := tc.Upgrade(sdkresource.TiDBClusterUpgradeOptions{
				Version: "v5.0.2",
				Timeout: 10 * time.Minute,
			})
			framework.ExpectNoError(err)
			framework.ExpectNoError(tc.SyncTiDBCluster())
			s, _ := tc.GetStatus(sdkresource.TiDB)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/tidb:v5.0.2"))
			s, _ = tc.GetStatus(sdkresource.TiKV)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/tikv:v5.0.2"))
			s, _ = tc.GetStatus(sdkresource.PD)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/pd:v5.0.2"))
			sts, err := f.KubeClientSetPack.ClientSet.AppsV1().StatefulSets(tc.Namespace()).Get(context.TODO(), fmt.Sprintf("tc-ticdc"), metav1.GetOptions{})
			framework.ExpectNoError(err)
			Expect(sts.Spec.Template.Spec.Containers[0].Image, "hub.pingcap.net/qa/ticdc:v5.0.2")
		})

		It("TiDBCluster update cluster should erase component version", func() {
			tc := f.CreateTiDBClusterWithTiDBHotfix(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})

			err := tc.Upgrade(sdkresource.TiDBClusterUpgradeOptions{
				Version: "v5.2.2",
				Timeout: 10 * time.Minute,
			})
			framework.ExpectNoError(err)
			framework.ExpectNoError(tc.SyncTiDBCluster())
			s, _ := tc.GetStatus(sdkresource.TiDB)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/tidb:v5.2.2"))
			s, _ = tc.GetStatus(sdkresource.TiKV)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/tikv:v5.2.2"))
			s, _ = tc.GetStatus(sdkresource.PD)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/pd:v5.2.2"))
			sts, err := f.KubeClientSetPack.ClientSet.AppsV1().StatefulSets(tc.Namespace()).Get(context.TODO(), fmt.Sprintf("tc-ticdc"), metav1.GetOptions{})
			framework.ExpectNoError(err)
			Expect(sts.Spec.Template.Spec.Containers[0].Image, "hub.pingcap.net/qa/ticdc:v5.2.2")
		})

		It("TiDBCluster update cluster with override components should work", func() {
			tc := f.CreateTiDBClusterWithTiDBHotfix(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})

			err := tc.Upgrade(sdkresource.TiDBClusterUpgradeOptions{
				Version: "v5.2.2",
				OverrideComponentsUpgradeOption: []sdkresource.TiDBClusterComponentUpgradeOption{
					{
						Target:  sdkresource.TiDB,
						Version: "v5.2.3",
					},
					{
						Target:  sdkresource.TiKV,
						Version: "v5.2.3",
					},
				},
				Timeout: 10 * time.Minute,
			})
			framework.ExpectNoError(err)
			framework.ExpectNoError(tc.SyncTiDBCluster())
			s, _ := tc.GetStatus(sdkresource.TiDB)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/tidb:v5.2.3"))
			s, _ = tc.GetStatus(sdkresource.TiKV)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/tikv:v5.2.3"))
			s, _ = tc.GetStatus(sdkresource.PD)
			Expect(s.Image).Should(Equal("hub.pingcap.net/qa/pd:v5.2.2"))
			sts, err := f.KubeClientSetPack.ClientSet.AppsV1().StatefulSets(tc.Namespace()).Get(context.TODO(), fmt.Sprintf("tc-ticdc"), metav1.GetOptions{})
			framework.ExpectNoError(err)
			Expect(sts.Spec.Template.Spec.Containers[0].Image, "hub.pingcap.net/qa/ticdc:v5.2.2")
		})

		It("TiDBCluster cluster cluster timeout should work", func() {
			tc := f.CreateTiDBClusterWithTiCDC(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})

			err := tc.Upgrade(sdkresource.TiDBClusterUpgradeOptions{
				Version: "v5.0.2",
				Timeout: 1 * time.Second,
			})
			Expect(pkgErrors.Cause(err)).Should(Equal(k8s.ErrUpgradeTimeout))
		})
	})

	Context("TiDBCluster.GetSpec", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-getspec")

		It("GetSpec should works for all components", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			Expect(f.GetTiDBClusterSpec(tc, sdkresource.TiDB).Replicas).Should(Equal(1))
			Expect(f.GetTiDBClusterSpec(tc, sdkresource.PD).Replicas).Should(Equal(1))
			Expect(f.GetTiDBClusterSpec(tc, sdkresource.TiKV).Replicas).Should(Equal(1))
		})
	})

	Context("TiDBCluster.GetStatus", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-getstatus")

		It("GetStatus should returns correct ReadyReplicas and Normal once the resource have been awaited", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			tidb := f.GetTiDBClusterStatus(tc, sdkresource.TiDB)
			pd := f.GetTiDBClusterStatus(tc, sdkresource.PD)
			tikv := f.GetTiDBClusterStatus(tc, sdkresource.TiKV)

			Expect(tidb.Phase).Should(Equal("Normal"))
			Expect(pd.Phase).Should(Equal("Normal"))
			Expect(tikv.Phase).Should(Equal("Normal"))

			Expect(tidb.Status.ReadyReplicas).Should(Equal(1))
			Expect(pd.Status.ReadyReplicas).Should(Equal(1))
			Expect(tikv.Status.ReadyReplicas).Should(Equal(1))
		})
	})

	Context("TiDBCluster with ngmonitoring deployed", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-ngm")

		It("ngm should deploy succeed", func() {
			tc := f.CreateTiDBClusterWithNGMonitorConfigured(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			pod, err := f.KubeClientSetPack.ClientSet.CoreV1().Pods(tc.Namespace()).Get(context.TODO(), "tc-ng-monitoring-0", metav1.GetOptions{})
			framework.ExpectNoError(err)
			Expect(pod.Status.Phase).Should(Equal(corev1.PodRunning))
			Expect(pod.Spec.Containers[0].Image).Should(Equal("registry-mirror.pingcap.net/pingcap/ng-monitoring:v5.4.1"))
		})
	})

	Context("TiDBCluster.Prometheus", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-prometheus")

		It("Prometheus-Client should compute metric QPS successfully once the cluster is ready", func() {
			tc := f.CreateTiDBClusterWithMonitorConfigured(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			url, err := tc.ServiceURL(sdkresource.Prometheus)
			framework.ExpectNoError(err)
			client, err := prometheus.NewClient(prometheus.Config{
				Address: url.String(),
			})
			framework.ExpectNoError(err)

			// QPS: sum(rate(tidb_executor_statement_total{tidb_cluster="testbed-tidbcluster-getstatus-jijbtjve-tc-ymcwmxhu"}[1m]))
			// Transaction OPS: sum(rate(tidb_session_transaction_duration_seconds_count{tidb_cluster="testbed-tidbcluster-getstatus-jijbtjve-tc-ymcwmxhu"}[1m])) by (type)
			// Up Store Count: sum(pd_cluster_status{tidb_cluster="testbed-tidbcluster-getstatus-jijbtjve-tc-ymcwmxhu", instance="tc-ymcwmxhu-pd-0", type="store_up_count"})
			prometheusTidbCluster := fmt.Sprintf("%s-%s", tc.Namespace(), tc.Name())
			q := fmt.Sprintf("sum(rate(tidb_executor_statement_total{tidb_cluster=\"%s\"}[1m]))", prometheusTidbCluster)
			qps, warnings, err := prometheus_api.NewAPI(client).Query(context.TODO(), q, time.Now())
			framework.ExpectNoError(err)

			framework.Logf("executed %s against Prometheus: warnings: %s", q, warnings)
			// Expect(len(warnings)).Should(Equal(0))

			Expect(qps).Should(BeAssignableToTypeOf(model.Vector{}))
			framework.Logf("result: %s", qps)
		})
	})

	Context("TiDBCluster.WatchOOM", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-watchoom")

		It("Should capture oom event", func() {
			tc := f.CreateTiDBClusterOOM(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			ctx, cancel := context.WithCancel(context.TODO())
			oomEventCh, err := tc.WatchOOM(ctx, sdkresource.TiDBClusterOOMWatchOptions{Target: sdkresource.TiKV})
			framework.ExpectNoError(err)

			// there isn't memory for tikv, so tikv pod will continuously OOM on bootstraping
			event := <-oomEventCh
			Expect(event.InstanceName).Should(ContainSubstring(sdkresource.TiKV.String()))
			cancel()
		})
	})

	Context("TiDBCluster.Artifact", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-artifact")

		It("Should artifact successfully", func() {
			tc := f.CreateTiDBClusterWithTiCDC(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			var data []string
			f.ArchiveTiDBCluster(tc.Handle(), f.Archiver(&data))
			Expect(len(data)).ShouldNot(BeZero())
		})
	})

	Context("TiDBCluster.SearchLog", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-searchlog")
		It("Searching logs on TiDB should work", func() {
			start := time.Now()
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})
			f.WaitForTiDBClusterReady(tc, framework.TestContext.ResourceWaitReadyTimeout)
			logs, err := tc.SearchLog(context.TODO(), sdkresource.TiDBClusterSearchLogOptions{
				Target:    sdkresource.TiDB,
				Levels:    []logsearch.LogLevel{logsearch.LogLevelInfo},
				StartTime: start,
				EndTime:   time.Now(),
				Patterns:  []string{"Welcome to TiDB"},
				Limit:     10,
			})
			framework.ExpectNoError(err)
			Expect(len(logs)).ShouldNot(Equal(0))
		})
		It("Searching logs on TiKV should work", func() {
			start := time.Now()
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})
			f.WaitForTiDBClusterReady(tc, framework.TestContext.ResourceWaitReadyTimeout)
			logs, err := tc.SearchLog(context.TODO(), sdkresource.TiDBClusterSearchLogOptions{
				Target:    sdkresource.TiKV,
				Levels:    []logsearch.LogLevel{logsearch.LogLevelInfo},
				StartTime: start,
				EndTime:   time.Now(),
				Patterns:  []string{"Welcome to TiKV"},
				Limit:     10,
			})
			framework.ExpectNoError(err)
			Expect(len(logs)).ShouldNot(Equal(0))
		})
		It("Searching logs on PD should work", func() {
			start := time.Now()
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})
			f.WaitForTiDBClusterReady(tc, framework.TestContext.ResourceWaitReadyTimeout)
			logs, err := tc.SearchLog(context.TODO(), sdkresource.TiDBClusterSearchLogOptions{
				Target:    sdkresource.PD,
				Levels:    []logsearch.LogLevel{logsearch.LogLevelInfo},
				StartTime: start,
				EndTime:   time.Now(),
				Patterns:  []string{"Welcome to Placement Driver (PD)"},
				Limit:     10,
			})
			framework.ExpectNoError(err)
			Expect(len(logs)).ShouldNot(Equal(0))
			framework.Logf("Show welcome logs: %+v", logs)
		})
		It("Searching logs on TiFlash should work", func() {
			start := time.Now()
			tc := f.CreateTiDBClusterWithTiflash(f.CreateTestbed())
			tc = f.GetTiDBCluster(tc.Handle())

			k8s.SetConfig(k8s.Config{
				TiDBCluster: k8s.TiDBClusterConfig{
					ReadyCheck: k8s.TiDBClusterReadyCheckConfig{
						PingDB: false,
					},
				},
			})
			f.WaitForTiDBClusterReady(tc, framework.TestContext.ResourceWaitReadyTimeout)
			logs, err := tc.SearchLog(context.TODO(), sdkresource.TiDBClusterSearchLogOptions{
				Target:    sdkresource.TiFlash,
				Levels:    []logsearch.LogLevel{logsearch.LogLevelInfo},
				StartTime: start,
				EndTime:   time.Now(),
				Patterns:  []string{"TiFlash build info"},
				Limit:     10,
			})
			framework.ExpectNoError(err)
			Expect(len(logs)).ShouldNot(Equal(0))
			framework.Logf("Show welcome logs: %+v", logs)
		})
	})

	Context("TiDBCluster.GetNodes", func() {
		f := framework.NewDefaultFramework("testbed-tidbcluster-getnodes")
		It("TiDBComponentNode.CleanData should work", func() {
			tc := f.CreateTiDBCluster(f.CreateTestbed())
			f.WaitForTiDBClusterReadyWithoutPing(tc, framework.TestContext.ResourceWaitReadyTimeout)

			createFilePath := "/var/lib/tidb/log/yelo"
			_, _, _, err := tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiDB,
				Peer:    0,
				Command: fmt.Sprintf("touch %s", createFilePath),
			})
			framework.ExpectNoError(err)

			tidbNodes, err := tc.GetNodes(sdkresource.TiDB)
			framework.ExpectNoError(err)
			for _, node := range tidbNodes {
				framework.ExpectNoError(node.CleanData())
			}
			framework.ExpectNoError(tc.WaitForReady(10 * time.Minute))

			_, _, _, err = tc.Exec(sdkresource.TiDBClusterExecOptions{
				Target:  sdkresource.TiDB,
				Peer:    0,
				Command: fmt.Sprintf("ls %s", createFilePath),
			})
			framework.ExpectError(err, "created file should have been deleted")
		})
	})
})