package experiment

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/juju/errors"
	"github.com/ngaut/log"
	"github.com/pingcap/schrodinger/config"
	"github.com/pingcap/schrodinger/manager/cat"
	catCtrl "github.com/pingcap/schrodinger/manager/controller/cat"
	testCtrl "github.com/pingcap/schrodinger/manager/controller/test"
	"github.com/pingcap/schrodinger/manager/test"
	"github.com/pingcap/schrodinger/pkg/notify"
	"github.com/pingcap/schrodinger/pkg/storage"
	"github.com/pingcap/schrodinger/pkg/types"
	"github.com/pingcap/schrodinger/util"
)

type endState string

const (
	endStateError    endState = "ERROR"
	endStateStopped           = "STOPPED"
	endStateFinished          = "FINISHED"
	endStateOK                = "OK"
)

// Operator is a interface to do misc tasks
type Operator interface {
	// AdjustTidbGc adjusts tidb gc
	AdjustTidbGc(tidbIP string) error

	// DeleteNamespace deletes namespace
	DeleteNamespace(namespace string) error
}

// Experiment is a set of test case.
type Experiment struct {
	sync.RWMutex
	wg sync.WaitGroup

	ctx    context.Context
	cancel context.CancelFunc

	process *types.ExperimentProcess

	cat   *cat.Cat
	tests map[string]*test.Test

	store    *storage.Storage
	catCtrl  *catCtrl.Controller
	testCtrl *testCtrl.Controller

	endState endState

	// errorCallback is callbacks when experiment error
	errorCallback []callbackFunc
	// finishCallback is callbacks  when experiment finished
	finishedCallback []callbackFunc

	destroying int32

	catNodeLabels  map[string]string
	testNodeLabels map[string]string

	notifiers map[string]notify.Notifier

	operator Operator
}

// NewExperiment creates experiment
func NewExperiment(
	ctx context.Context,
	meta *types.ExperimentProcess,
	store *storage.Storage,
	catCtrl *catCtrl.Controller,
	testCtrl *testCtrl.Controller,
	notifiers map[string]notify.Notifier,
	operator Operator,
) (*Experiment, error) {
	ctx, cancel := context.WithTimeout(ctx, meta.Config.Misc.Timeout)
	e := &Experiment{
		ctx:    ctx,
		cancel: cancel,

		process: meta,

		store:    store,
		catCtrl:  catCtrl,
		testCtrl: testCtrl,

		endState: endStateOK,

		tests: make(map[string]*test.Test),

		notifiers: notifiers,

		operator: operator,
	}

	catNodeLabels := make(map[string]string)
	if e.process.Config != nil && e.process.Config.Cat != nil {
		catNodeLabels = util.GenLabels(e.process.Config.Cat.LabelStr)
	}
	e.catNodeLabels = catNodeLabels

	testNodeLabels := make(map[string]string)
	if e.process.Config != nil ||
		e.process.Config.Tests != nil {
		testNodeLabels = util.GenLabels(e.process.Config.Tests.LabelStr)
	}
	e.testNodeLabels = testNodeLabels

	if !e.isNewExperiment() {
		return e, nil
	}

	index, err := e.store.IncBoxExperimentCount(e.process.BoxID)
	if err != nil {
		e.setStage(types.ExperimentStateAbnormal)
		e.setStatusMessage(err.Error())
		// TODO: add retry and notify
		e.store.UpdateExperimentProcess(e.process)
		return nil, errors.Trace(err)
	}

	e.process.BoxIndex = index
	e.process.Labels = genExperimentLabels(e.getTestConfigLabel())
	return e, nil
}

// Start starts experiment
func (experiment *Experiment) Start() error {
	// when recover abnormal experiment,
	if experiment.isStateAbnormal() {
		return nil
	}

	experiment.Lock()
	defer func() {
		experiment.save()
		experiment.Unlock()
	}()

	if experiment.isNewExperiment() {
		experiment.process.SetStartTime()
	}

	experiment.registerDefaultCallback()

	err := experiment.createTestProcesses()
	if err != nil {
		log.Errorf("experiment %s(%d) create test processes error %v", experiment.name(), experiment.id(), err)
		experiment.setState(types.ExperimentStateAbnormal)
		experiment.setStatusMessage(err.Error())
		experiment.onError()
		return errors.Trace(err)
	}

	experiment.save()

	log.Infof("start experiment %s(%d)", experiment.name(), experiment.id())

	if !experiment.isRunning() {
		if !experiment.process.Config.WithoutCat {
			if err := experiment.prepare(); err != nil {
				// if it was canceled by destroy. it is ok to call destroyed function to clean resources
				if strings.Contains(err.Error(), "context canceled") {
					experiment.onDestroyed()
					return nil
				}

				log.Errorf("prepare experiment %s(%d) failed %s", experiment.name(), experiment.id(), errors.ErrorStack(err))
				experiment.setState(types.ExperimentStateAbnormal)
				experiment.setStatusMessage(err.Error())
				experiment.onError()
				return errors.Trace(err)
			}
		}

		log.Infof("start experiment %s(%d) success, now it is running", experiment.name(), experiment.id())
		experiment.setStage(types.ExperimentStageRunning)
	} else {
		if experiment.process.Config != nil &&
			!experiment.process.Config.WithoutCat {
			if err := experiment.startCat(); err != nil {
				experiment.setState(types.ExperimentStateAbnormal)
				experiment.setStatusMessage(fmt.Sprintf("recover experiment failed, error: %v", err))
				experiment.onError()
				return errors.Trace(err)
			}
		}
	}

	go experiment.runTests()

	go experiment.monitor()

	return nil
}

// Destroy destroys experiment
func (experiment *Experiment) Destroy() error {
	// use atomic variable control this function only one is running
	if !atomic.CompareAndSwapInt32(&experiment.destroying, 0, 1) {
		return errors.MethodNotAllowedf("experiment %s is destroying", experiment.name())
	}

	if experiment.IsDestroyed() {
		return errors.MethodNotAllowedf("experiment %s was already destroyed", experiment.name())
	}

	defer func() {
		experiment.save()
		atomic.CompareAndSwapInt32(&experiment.destroying, 1, 0)
	}()

	if experiment.isStateAbnormal() {
		log.Infof("dry destroy experiment %s(%d)", experiment.name(), experiment.id())
		if err := experiment.onDestroyed(); err != nil {
			return errors.Trace(err)
		}
		return nil
	}

	log.Infof("destroy experiment %s(%d)", experiment.name(), experiment.id())

	experiment.cancel()
	experiment.wg.Wait()
	return nil
}

// Annotate assigns experiment process annotation field
func (experiment *Experiment) Annotate(annotation string, author string) {
	experiment.process.Annotation = annotation
	experiment.process.Author = author
	experiment.save()
}

// ReplaceCat replaces cat
func (experiment *Experiment) ReplaceCat(cat *types.Cat) error {
	experiment.Lock()
	defer func() {
		experiment.save()
		experiment.Unlock()
	}()

	if experiment.process == nil {
		log.Errorf("the process of experiment %s can not be empty", experiment.name())
		return errors.Errorf("the process of experiment %s can not be empty", experiment.name())
	}

	if experiment.process.Config == nil || experiment.process.Config.Cat == nil {
		log.Errorf("the cat config of the experiment %s can not be empty", experiment.name())
		return errors.Errorf("the cat config of the experiment %s can not be empty", experiment.name())
	}

	if experiment.process.Cat == nil {
		log.Errorf("the cat of experiment %s can not be empty", experiment.name())
		return errors.Errorf("the cat of experiment %s can not be empty", experiment.name())
	}

	experiment.process.CatCount++

	namespace := genCatNamespace(experiment.process)
	catName := util.GenCatName(experiment.process.Name, experiment.process.CatCount)

	catProcess := types.GenCatProcess(catName, namespace, cat)
	catProcess.ExperimentName = experiment.name()
	catProcess.BoxID = experiment.BoxID()
	catProcess.BoxName = experiment.BoxName()
	catProcess.Labels = experiment.process.Cat.Labels
	if err := experiment.store.CreateCatProcess(catProcess); err != nil {
		return errors.Trace(err)
	}

	log.Infof("[experiment %s(%d)] start to replace cat [namespace : %s] labels %+v", experiment.name(), experiment.id(), catProcess.Namespace, catProcess.Labels)
	if err := experiment.catCtrl.DestroyCat(experiment.process.Cat.ID); err != nil && !strings.Contains(err.Error(), "not found") {
		return errors.Trace(err)
	}

	experiment.process.Config.Cat = cat
	experiment.process.Cat = catProcess
	c, err := experiment.catCtrl.StartCat(experiment.ctx, catProcess)
	if err != nil {
		log.Errorf("[experiment %s(%d)] start cat failed %v", experiment.name(), experiment.id(), err)
		return errors.Trace(err)
	}

	experiment.cat = c

	experiment.replaceCatForTests(catProcess)

	log.Infof("[experiment %s(%d)] replace cat [namespace : %s] success", experiment.name(), experiment.id(), c.Namespace())
	return nil
}

// StartTest starts test
// mark test online, and start test
// the test should be idempotent
func (experiment *Experiment) StartTest(name string) error {
	experiment.Lock()
	defer func() {
		experiment.save()
		experiment.Unlock()
	}()

	log.Infof("[experiment %s(%d)] start test %s", experiment.name(), experiment.id(), name)
	t, exist := experiment.tests[name]
	if !exist {
		log.Errorf("[experiment %s(%d)] start test %s failed, test not exist", experiment.name(), experiment.id(), name)
		return errors.NotFoundf("test %s", name)
	}

	if !t.IsOffline() {
		log.Errorf("[experiment %s(%d)] start test %s failed,test marked not as %s", experiment.name(), experiment.id(), name, t.Mark())
		return errors.NotSupportedf("marked %s", t.Mark())
	}

	t.MarkOnline()
	err := t.Start(experiment.ctx)
	if err != nil {
		log.Infof("[experiment %s(%d)] start test %s failed", experiment.name(), experiment.id(), name)
		return errors.Trace(err)
	}

	log.Infof("[experiment %s(%d)] start test %s success", experiment.name(), experiment.id(), name)
	return nil
}

// StopTest stops test
// mark test as offline, and stop test
func (experiment *Experiment) StopTest(name string) error {
	experiment.Lock()
	defer func() {
		experiment.save()
		experiment.Unlock()
	}()

	log.Infof("[experiment: %s(%d)]start to stop test %s", experiment.name(), experiment.id(), name)
	t, exist := experiment.tests[name]
	if !exist {
		log.Errorf("[experiment: %s(%d)]start to stop test %s failed, test %s not exist", experiment.name(), experiment.id(), name, name)
		return errors.NotFoundf("test %s", name)
	}

	t.MarkOffline()
	err := t.Stop(experiment.ctx)
	if err != nil {
		log.Errorf("[experiment: %s(%d)]start to stop test %s failed, error %v", experiment.name(), experiment.id(), name, err)
		return errors.Trace(err)
	}

	log.Infof("[experiment: %s(%d)] stop test %s success", experiment.name(), experiment.id(), name)
	return nil
}

// RemoveTest removes test from experiment
// first: mark the test in Experiment as removed
// second: mark the test in ExperimentProcess as removed
// third: destroy test
// why we not delete the  Experiment? because we need to recode this.
// when recover manager this will be done. and when monitor, we should count this
func (experiment *Experiment) RemoveTest(name string) error {
	experiment.Lock()

	defer func() {
		t, exist := experiment.tests[name]
		if exist {
			t.MarkRemoved()
		}

		experiment.save()
		experiment.Unlock()
	}()

	return experiment.destroyTest(name)
}

func (experiment *Experiment) monitor() {
	experiment.wg.Add(1)
	ticker := time.NewTicker(config.GetGlobalConfig().Experiment.SyncInterval.Duration)
	defer func() {
		log.Infof("monitor %s stop", experiment.name())
		ticker.Stop()
		experiment.wg.Done()
	}()

	end := false
	for !end {
		select {
		case <-experiment.ctx.Done():
			if experiment.ctx.Err() == context.DeadlineExceeded {
				experiment.endState = endStateFinished
			} else {
				experiment.endState = endStateStopped
			}
		case <-ticker.C:
			experiment.syncStatus()
		}
		if experiment.isEnd() {
			end = true
		}
	}

	if experiment.isErr() {
		select {
		case <-experiment.ctx.Done():
			if experiment.ctx.Err() == context.DeadlineExceeded {
				experiment.endState = endStateFinished
			} else {
				experiment.endState = endStateStopped
			}
		default:
		}
	}

	experiment.onEnd()
	return
}

func (experiment *Experiment) onEnd() {
	defer experiment.save()

	if experiment.endState == endStateError {
		experiment.onError()
	} else if experiment.endState == endStateFinished {
		experiment.onFinished()
	} else {
		experiment.onDestroyed()
	}
}

func (experiment *Experiment) onError() {
	experiment.setState(types.ExperimentStateAbnormal)

	defer func() {
		experiment.save()
		for _, cb := range experiment.errorCallback {
			cb(experiment)
		}
	}()

	experiment.stopTests()
}

func (experiment *Experiment) onFinished() {
	defer func() {
		experiment.process.SetEndTime()
		experiment.save()
		for _, cb := range experiment.finishedCallback {
			cb(experiment)
		}
	}()

	experiment.setStage(types.ExperimentStageFinishing)

	experiment.process.BeDestroyed()
	experiment.destroy()

	experiment.setStage(types.ExperimentStageFinished)
}

// TODO: this method should only be allowed to run once at a time
func (experiment *Experiment) onDestroyed() error {
	if err := experiment.destroy(); err != nil {
		log.Errorf("destroy experiment %s failed, error: %v", experiment.name(), err)
		return errors.Trace(err)
	}
	experiment.process.SetEndTime()
	experiment.process.BeDestroyed()
	return nil
}

func (experiment *Experiment) prepare() error {
	experiment.setStage(types.ExperimentStagePreparing)

	err := experiment.startCat()
	if err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (experiment *Experiment) destroy() error {
	log.Infof("destroy experiment %s(%d)", experiment.name(), experiment.id())
	err := experiment.destroyTests()
	if err != nil {
		log.Errorf("[experiment %s]destroy tests failed", experiment.name())
		return errors.Trace(err)
	}

	if experiment.process == nil ||
		experiment.process.Config == nil ||
		experiment.process.Config.WithoutCat {
		return nil
	}

	if !experiment.shouldDestroyCat() {
		// When sometimes create cat failed. cat is nil
		if experiment.process.Cat != nil {
			experiment.unClaimCat()
		}
		return nil
	}

	err = experiment.destroyCat()
	if err != nil {
		log.Errorf("[experiment %s]destroy cat failed, error %v", experiment.name(), err)
		// TODO: we should abstract a pkg to distinguish the type of the error
		if strings.Contains(err.Error(), "not found") ||
			strings.Contains(err.Error(), "already stopped") {
			return nil
		}
		return errors.Trace(err)
	}

	return nil
}

func (experiment *Experiment) destroyTest(name string) error {
	defer func() {
		experiment.save()
	}()

	log.Infof("[experiment: %s(%d)] start to destory test %s", experiment.name(), experiment.id(), name)

	t, exist := experiment.tests[name]
	if !exist {
		log.Errorf("[experiment: %s(%d)] destroy test %s failed, test not exist", experiment.name(), experiment.id(), name)
		return errors.NotFoundf("test %s", name)
	}

	if err := t.Destroy(experiment.ctx); err != nil {
		log.Errorf("[experiment: %s(%d)] destroy test %s failed, error: %v", experiment.name(), experiment.id(), name, err)
		return errors.Trace(err)
	}

	log.Infof("[experiment: %s(%d)] destroy test %s success", experiment.name(), experiment.id(), name)
	return nil
}

func (experiment *Experiment) startCat() error {
	var (
		err        error
		catProcess *types.CatProcess
		exist      bool
	)

	if experiment.process.Cat == nil {
		if experiment.process.Config.WithoutCat {
			return nil
		}

		if experiment.process.Config.Cat == nil {
			return errors.New("config cat is nil")
		}

		if util.IsEmpty(experiment.process.Config.Cat.SelectedCat) {
			namespace := genCatNamespace(experiment.process)
			catName := util.GenCatName(experiment.process.Name, experiment.process.CatCount)

			catProcess = types.GenCatProcess(catName, namespace, experiment.process.Config.Cat)
			catProcess.ExperimentName = experiment.name()
			catProcess.BoxID = experiment.BoxID()
			catProcess.BoxName = experiment.BoxName()
			if err = experiment.store.CreateCatProcess(catProcess); err != nil {
				return errors.Trace(err)
			}
		} else {
			catProcess, exist = experiment.store.GetCatProcessByName(experiment.process.Config.Cat.SelectedCat)
			if !exist {
				return errors.NotFoundf("cat %s", experiment.process.Config.Cat.SelectedCat)
			}

			if !catProcess.IsRunning() || !catProcess.IsStateNormal() {
				return errors.Errorf("cat %s selected is abnormal or not running", experiment.process.Config.Cat.SelectedCat)
			}
		}
		experiment.process.Cat = catProcess
	} else {
		// set cat process of box from storage
		catProcess, exist = experiment.store.GetCatProcessByName(experiment.process.Cat.Name)
		if !exist {
			return errors.NotFoundf("cat %s", experiment.process.Cat.Name)
		}
		experiment.process.Cat = catProcess
	}

	experiment.claimCat()
	experiment.save()

	experiment.cat, err = experiment.catCtrl.StartCat(experiment.ctx, experiment.process.Cat)
	if err != nil {
		if errors.IsAlreadyExists(err) {
			return nil
		}

		return errors.Trace(err)
	}

	return nil
}

func (experiment *Experiment) destroyCat() error {
	if experiment.process.Cat == nil {
		log.Infof("experiment %s(%d) cat is nil, do not need destroy cat", experiment.name(), experiment.id())
		return nil
	}
	return errors.Trace(experiment.catCtrl.DestroyCat(experiment.process.Cat.ID))
}

func (experiment *Experiment) runTests() error {
	log.Infof("[experiment %s] start to run prepare tests", experiment.name())
	if err := experiment.runTestsAction(types.PrepareTest, true); err != nil {
		log.Errorf("[experiment %s] run prepare tests failed: %v", experiment.name(), err)
		return errors.Trace(err)
	}

	log.Infof("[experiment %s] start to run general tests", experiment.name())
	if err := experiment.runTestsAction(types.GeneralTest, experiment.process.Config.Tests.InOrder); err != nil {
		log.Errorf("[experiment %s] run real tests failed: %v", experiment.name(), err)
		return errors.Trace(err)
	}
	return nil
}

func (experiment *Experiment) runTestsAction(testType types.TestType, shouldWait bool) error {
	runAction := func(t *types.TestProcess) error {
		if err := experiment.runTest(t); err != nil {
			if !errors.IsAlreadyExists(err) {
				log.Errorf("[experiment %s] run [test: %s] failed: %v", experiment.name(), t.Name, err)
				experiment.endState = endStateError
				experiment.setStatusMessage(fmt.Sprintf("error on run test: %v", err))
				return errors.Trace(err)
			}
			log.Infof("[experiment %s] test %s is already run", experiment.name(), t.Name)
		}

		if shouldWait {
			if err := waitTillFinish(experiment.ctx, experiment.tests[t.Name]); err != nil {
				log.Errorf("[experiment %s] waiting for [test: %s] finished failed: %v", experiment.name(), t.Name, err)
				return errors.Trace(err)
			}

			log.Infof("[experiment: %s] test %s finished, destroy it", experiment.name(), t.Name)
			if err := experiment.destroyTest(t.Name); err != nil {
				log.Errorf("[experiment: %s] destroy test %s failed, error: %v", experiment.name(), t.Name, err)
				return nil
			}
		}
		return nil
	}

	for _, t := range experiment.process.Tests {
		if t.Template.Type != testType {
			continue
		}

		// to run running test first
		if t.Status == types.TestStateRunning || t.Status == types.TestStateCreating {
			if err := runAction(t); err != nil {
				return errors.Trace(err)
			}
		}
	}

	for _, t := range experiment.process.Tests {
		if t.Template.Type != testType {
			continue
		}

		if _, ok := experiment.tests[t.Name]; ok {
			continue
		}

		if t.Status == types.TestStateDestroyed {
			experiment.tests[t.Name] = experiment.testCtrl.NewTest(t)
			continue
		}

		if err := runAction(t); err != nil {
			return errors.Trace(err)
		}
	}

	return nil
}

func (experiment *Experiment) createTestProcesses() error {
	for _, test := range experiment.getTests() {
		err := experiment.createTestProcess(&test, experiment.process.Labels)
		if err != nil && strings.Contains(err.Error(), "not found") {
			return errors.Trace(err)
		}

		if err != nil {
			log.Warningf("[experiment %s(%d)]test template %v not exist", experiment.name(), experiment.id(), test)
			continue
		}
	}

	return nil
}

func (experiment *Experiment) createTestProcess(testConfig *types.TestConfig, labels map[string]string) error {
	if _, exist := experiment.process.Tests[testConfig.TestName]; exist {
		return nil
	}

	testTemplate, err := experiment.store.GetTestTemplate(testConfig.TestTemplateID)
	if err != nil {
		return errors.Trace(err)
	}

	// TODO: need to uniform with create CatProcess
	testProc := &types.TestProcess{
		BoxID:        experiment.process.BoxID,
		ExperimentID: experiment.process.ID,
		Namespace:    experiment.getTestNamespace(),
		Template:     testTemplate,
		OverrideArgs: testConfig.OverrideArgs,
		Status:       types.TestStatePending,
		Name:         testConfig.TestName,
		Mark:         types.MarkOnline,
		Meta: &types.Meta{
			Labels: labels,
		},
	}

	if err := experiment.store.CreateTestProcess(testProc); err != nil {
		return errors.Trace(err)
	}
	experiment.process.Tests[testConfig.TestName] = testProc

	return nil
}

func (experiment *Experiment) replaceCatForTests(catProcess *types.CatProcess) {
	for _, test := range experiment.process.Tests {
		test.Cat = catProcess
	}
}

func (experiment *Experiment) runTest(testProc *types.TestProcess) error {
	if !experiment.process.IsRunning() {
		return errors.MethodNotAllowedf("[experiment: %s] is not running", experiment.name())
	}

	_, ok := experiment.tests[testProc.Template.Name]
	if ok {
		return errors.AlreadyExistsf("[experiment: %s][test: %s]", experiment.name(), testProc.Template.Name)
	}

	testProc.Cat = experiment.process.Cat

	tp, err := experiment.testCtrl.StartTest(testProc)
	experiment.tests[tp.Name()] = tp
	if err != nil {
		log.Errorf("[experiment: %s][test: %s] start failed: %v", experiment.name(), tp.Name(), err)
		return errors.Trace(err)
	}

	log.Infof("add [test: %s] to [experiment: %s]", tp.Name(), experiment.name())
	return nil
}

func (experiment *Experiment) stopTests() {
	for _, t := range experiment.tests {
		if err := experiment.testCtrl.StopTest(t.Identify()); err != nil {
			log.Errorf("[experiment: %s][case: %s] stop failed: %v",
				experiment.name(), t.Name(), err)
		}
	}
	log.Infof("[experiment: %s] stop case - stop cases successfully", experiment.name())
}

func (experiment *Experiment) destroyTests() error {
	namespace := experiment.getTestNamespace()
	log.Infof("destroy namespace %s", namespace)

	err := experiment.operator.DeleteNamespace(namespace)
	if err != nil && !strings.Contains(err.Error(), "not found") {
		return errors.Annotatef(err, "[namespace: %s] delete cluster namespace", namespace)
	}

	log.Infof("destroy namespace %s success", namespace)

	return nil
}

// TODO: need to refine save to ignore already removed test
func (experiment *Experiment) save() {
	report := reportInformation(experiment)
	if apps, ok := experiment.store.GetAPPProcessByExperimentID(experiment.id()); ok {
		report.APPs = apps.GetReports()
	}
	experiment.process.Report = report

	if err := experiment.store.UpdateExperimentProcess(experiment.process); err != nil {
		log.Errorf("[experiment %d] save experiment error %v", experiment.id(), err)
	}
}

// syncStatus syncs status
func (experiment *Experiment) syncStatus() {
	experiment.Lock()
	defer experiment.Unlock()
	log.Debugf("start sync experiment %s(%d)", experiment.name(), experiment.id())
	experiment.endState = endStateOK
	finishedCount := 0
	for _, cs := range experiment.tests {
		if cs.IsRemoved() {
			finishedCount++
			continue
		}

		// offline test do not need sync status
		if cs.IsOffline() {
			continue
		}

		// we will destroy the test when to run test serially and the test is finished
		if cs.IsDestroy() {
			cs.SetReport(genTestReport(cs.Name(), experiment))
			finishedCount++
			continue
		}

		cs.SyncStatus(experiment.ctx)

		cs.SetReport(genTestReport(cs.Name(), experiment))

		if cs.IsError() {
			experiment.endState = endStateError
		}

		if cs.IsFinished() {
			finishedCount++
		}
	}

	if finishedCount >= len(experiment.process.Config.Tests.Tests) {
		experiment.endState = endStateFinished
	}

	// sync the status of cat
	if experiment.cat != nil {
		// TODO: we should monitor the status of the cat based on changes in cases
		// eg: if we kill a `tikv` in a case, our monitor should get this information and adjust our monitoring strategy.
		healthy, err := experiment.cat.Health()
		if !healthy && !util.CheckIsNetworkError(err) {
			if experiment.process.Config != nil &&
				experiment.process.Config.Cat != nil &&
				experiment.process.Config.Cat.Monitor {
				experiment.endState = endStateError
				experiment.setStatusMessage(err.Error())
			}
		}
	}

	experiment.save()
}

func (experiment *Experiment) name() string {
	return experiment.process.Name
}

func (experiment *Experiment) id() int64 {
	return experiment.process.ID
}

func (experiment *Experiment) getTests() types.TestConfigs {
	var tests types.TestConfigs
	if experiment.process.Config.Tests != nil {
		tests = experiment.process.Config.Tests.Tests
	}
	return tests
}

func (experiment *Experiment) getTestConfigLabel() string {
	if experiment.process.Config.Tests != nil {
		return experiment.process.Config.Tests.LabelStr
	}

	return ""
}

func (experiment *Experiment) getTestNamespace() string {
	return experiment.process.Namespace
}

func (experiment *Experiment) shouldDestroyCat() bool {
	return experiment.process.Config.Misc.DestroyTidbCluster
}

// IsFinished returns true if experiment is finished
func (experiment *Experiment) IsFinished() bool {
	return experiment.process.IsFinished()
}

// IsDestroyed returns true if experiment is handle
func (experiment *Experiment) IsDestroyed() bool {
	return experiment.process.IsDestroyed()
}

func (experiment *Experiment) isEnd() bool {
	return experiment.endState == endStateError || experiment.endState == endStateFinished || experiment.endState == endStateStopped
}

func (experiment *Experiment) isErr() bool {
	return experiment.endState == endStateError
}

// isStateAbnormal returns true if experiment is finished
func (experiment *Experiment) isStateAbnormal() bool {
	return experiment.process.IsStateAbnormal()
}

func (experiment *Experiment) setState(state types.ExperimentState) {
	if state == types.ExperimentStateAbnormal {
		experiment.process.Ftime = time.Now().Format(util.TimeFormat)
	}
	experiment.process.State = state
}

func (experiment *Experiment) setStage(stage types.ExperimentStage) {
	experiment.process.Stage = stage
}

func (experiment *Experiment) setStatusMessage(status string) {
	experiment.process.StatusMessage = status
}

func (experiment *Experiment) registerDefaultCallback() {
	experiment.registerCallback("adjustgc", endStateError)
	experiment.registerCallback("slack", endStateError)
	experiment.registerCallback("jira", endStateError)

	experiment.registerCallback("slack", endStateFinished)
	experiment.registerCallback("jira", endStateFinished)
}

func (experiment *Experiment) registerCallback(name string, state endState) {
	cb, ok := callbackFuncs[name]
	if !ok {
		return
	}

	switch state {
	case endStateError:
		experiment.errorCallback = append(experiment.errorCallback, cb)
	case endStateFinished:
		experiment.finishedCallback = append(experiment.finishedCallback, cb)
	}
}

// TODO: refine cat code to use cat.SetOwner. we should not direct set cat owner in experiment
func (experiment *Experiment) claimCat() {
	experiment.process.Cat.Owner = experiment.process.Name
	experiment.process.Cat.ExperimentName = experiment.process.Name
	experiment.process.Cat.BoxName = experiment.BoxName()
	experiment.process.Cat.BoxID = experiment.BoxID()
}

func (experiment *Experiment) unClaimCat() {
	experiment.process.Cat.Owner = ""
	experiment.process.Cat.ExperimentName = ""
	experiment.process.Cat.BoxName = ""
}

func (experiment *Experiment) getExperimentIndexInBox() int64 {
	return experiment.process.BoxIndex
}

func (experiment *Experiment) isNewExperiment() bool {
	return experiment.process.Stage == types.ExperimentStagePending
}

func (experiment *Experiment) isRunning() bool {
	return experiment.process.Stage == types.ExperimentStageRunning
}

// BoxID returns box id
func (experiment *Experiment) BoxID() int64 {
	return experiment.process.BoxID
}

// Type returns the type of the experiment
func (experiment *Experiment) Type() types.BoxType {
	box, exist := experiment.store.GetBox(experiment.process.BoxID)
	if !exist {
		log.Errorf("[experiment %s] can not found box %d", experiment.process.Name, experiment.process.BoxID)
		return types.BoxTypeGeneral
	}

	return box.Misc.Type
}

// BoxName returns the name of the box
func (experiment *Experiment) BoxName() string {
	return experiment.process.BoxName
}

func (experiment *Experiment) nodeLabelsForUpdate() map[string]string {
	box, exist := experiment.store.GetBox(experiment.process.BoxID)
	if !exist {
		log.Errorf("[experiment %s] can not found box %d", experiment.process.Name, experiment.process.BoxID)
		return nil
	}

	return box.NodeLabelsForUpdate
}

func (experiment *Experiment) emptyNodeBoxLabelForUpdate() error {
	box, exist := experiment.store.GetBox(experiment.process.BoxID)
	if !exist {
		log.Errorf("[experiment %s] can not found box %d", experiment.process.Name, experiment.process.BoxID)
		return errors.Errorf("[experiment %s] can not found box %d", experiment.process.Name, experiment.process.BoxID)
	}

	box.EmptyNodeLabelsForUpdate()

	if err := experiment.store.UpdateBox(box); err != nil {
		log.Errorf("[experiment: %s] update box failed", experiment.name())
		return errors.Trace(err)
	}

	return nil
}

func (experiment *Experiment) verify() error {
	if experiment == nil {
		return errors.New("experiment can not be nil")
	}

	return nil
}

// GetProcess gets experiment process
func (experiment *Experiment) GetProcess() *types.ExperimentProcess {
	return experiment.process.Clone()
}