package core

import (
	"context"
	"errors"
	"fmt"
	"go.uber.org/zap"
	"sort"
	"sync"
	"sync/atomic"
	"time"

	"github.com/guo-shaoge/tiflash-autoscale/pkg/config"
	"github.com/guo-shaoge/tiflash-autoscale/pkg/meta"
	"github.com/guo-shaoge/tiflash-autoscale/pkg/metric"
	"github.com/guo-shaoge/tiflash-autoscale/pkg/util/log/logger"
	k8sapiv1 "k8s.io/api/core/v1"
)

type PodMgr struct {
	ctx    context.Context
	logger *zap.Logger

	configMgr *ConfigMgr

	sharePool       *SharePool
	vipPool         *VIPPool
	warmupPool      *WarmupPool
	unassigningPool *UnassigningPool

	// <podName, lastLostTs>
	lostPods map[string]time.Time

	k8sMgr *K8sMgr

	failCnt atomic.Int32
}

func NewPodMgr(ctx context.Context, k8sMgr *K8sMgr, configMgr *ConfigMgr, parentLogger *zap.Logger) *PodMgr {
	res := &PodMgr{
		ctx:       ctx,
		logger:    parentLogger.With(zap.String(logger.LogIDMod, "podMgr")),
		configMgr: configMgr,

		sharePool:       NewSharePool(),
		vipPool:         NewVIPPool(),
		warmupPool:      NewWarmupPool(),
		unassigningPool: NewUnassigningPool(),

		lostPods: make(map[string]time.Time),

		k8sMgr: k8sMgr,
	}
	return res
}

func (p *PodMgr) startBgWorker(wg *sync.WaitGroup) {
	wg.Add(2)
	go func() {
		p.podScannerLoop()
	}()
	go func() {
		p.warmupLoop()
	}()
}

// The function try best to return num pods. If there are not enough pods in the pool,
// it will return at least one pod and increment the atomic variable failCnt.
// A background thread will then request new pods from k8s based on the value of failCnt.
// NOTE: always return pods that are Running or Sharing.
func (p *PodMgr) SyncTakePods(num uint, tenantName string, enableShare bool,
	sourceMod string, traceId string) []*meta.PodDesc {
	logger := p.logger.With(zap.String(logger.LogIDMod, sourceMod)).With(zap.String(logger.LogIDTrace, traceId))
	res := make([]*meta.PodDesc, 0, num)

	logger.Info("SyncTakePods begin")
	var shareCnt int
	defer func(resPtr *[]*meta.PodDesc) {
		tmpRes := *resPtr
		logger.Info("SyncTakePods finish", zap.Any("want", num), zap.Any("return", len(tmpRes)),
			zap.Any("shareCnt", shareCnt), zap.Any("warmupCnt", len(tmpRes)-shareCnt), zap.Any("enableShare", enableShare))
	}(&res)

	if enableShare {
		// take pods from share pool.
		shareRes := p.takePodsFromSharePool(num, tenantName, p.configMgr.GetConfig(), logger)
		shareCnt = len(shareRes)
		res = append(res, shareRes...)

		if len(res) >= int(num) {
			return res
		}
	}

	newNum := num - uint(shareCnt)
	if newNum > 0 {
		// TODO maybe also check UnassignPool for UnassignedPod.
		// So no need to wait PodScanner to wrok.
		// take pods from warmup pool.
		newPods := p.syncRemovePodsFromWarmupPool(tenantName, newNum, enableShare, logger)
		res = append(res, newPods...)
	}

	return res
}

// 1. remove from warmup pool
// 2. call grpc to setup tiflash process
// 3. call pod.Ref() to change status: stopped -> running
// 4. move it into share pool or vip pool
func (p *PodMgr) syncRemovePodsFromWarmupPool(tenantName string, num uint,
	enableShare bool, logger *zap.Logger) []*meta.PodDesc {
	filter := func(pod *meta.PodDesc) bool {
		return pod.IsUnassigned()
	}
	uncheckedPods := p.warmupPool.SyncRemovePodsByNum(num, filter, p, logger)

	grpcPods := make([]*meta.PodDesc, 0, len(uncheckedPods))
	errPods := []*meta.PodDesc{}

	// check status, all pods taken from warmup pool should be Stopped.
	for _, pod := range uncheckedPods {
		if pod.IsUnassigned() {
			grpcPods = append(grpcPods, pod)
		} else {
			errPods = append(errPods, pod)
		}
	}
	if len(errPods) != 0 {
		logger.Info("got pods from WarmupPool whose status is not Stopped, will ignore these pods",
			zap.Any("errPods", len(errPods)), zap.Any("grpcPods", len(grpcPods)))
	}

	// prepare grpc arguments.
	curClusterConfig := p.configMgr.GetConfig().GetClusterConfig()
	pdAddr := *curClusterConfig.PDAddr
	tiflashVer := fmt.Sprintf("%v", *curClusterConfig.ConfigVersion)

	okPods := []*meta.PodDesc{}
	var errPodsMu sync.Mutex
	var okPodsMu sync.Mutex
	wg := &sync.WaitGroup{}

	for _, pod := range grpcPods {
		wg.Add(1)
		go func() {
			defer wg.Done()
			resp, err := pod.SendAssignRPC(pdAddr, tiflashVer, logger)
			// TODO make it a func.
			if err == nil {
				if resp == nil {
					err = errors.New("AssignRPC resp is nil")
				} else if resp.HasErr {
					err = fmt.Errorf("AssignRPC resp.HasErr, errInfo: %v", resp.HasErr, resp.ErrInfo)
				} else {
					// TODO
				}
			}
			if err == nil {
				fromStatus, toStatus, err1 := pod.Ref(tenantName, "syncRemovePodsFromWarmupPool: new assigned pod")
				if err1 != nil {
					err = fmt.Errorf("got error when ref pod(%v to %v, err: %v)",
						fromStatus.ToString(), toStatus.ToString(), err1)
				}
			}

			if err != nil || !pod.IsRunning() {
				errPodsMu.Lock()
				errPods = append(errPods, pod)
				errPodsMu.Unlock()
				logger.Error("got error when trying to assign pod",
					zap.Any("pod", pod.GetName()), zap.Error(err), zap.String("status", pod.GetStatusStr()))
			} else {
				okPodsMu.Lock()
				okPods = append(okPods, pod)
				okPodsMu.Unlock()
			}
		}()
	}
	wg.Wait()

	// all pods are running
	var msg string
	if enableShare {
		_, msg = p.sharePool.AddPods(okPods)
	} else {
		_, msg = p.vipPool.AddPods(okPods)
	}
	if len(msg) != 0 {
		logger.Error("got duplicated pods when adding to SharePool/VIPPool", zap.String("msg", msg))
	}

	// handle error pods, change it to error and put to stopping pool.
	if len(errPods) > 0 {
		p.rollbackPods(errPods, "when take pods from WarmupPool", logger)
		errPods = nil
	}
	return okPods
}

func (p *PodMgr) rollbackPods(errPods []*meta.PodDesc, msg string, logger *zap.Logger) {
	var podsStr string
	for _, pod := range errPods {
		pod.ToLogicalError()
		podsStr += pod.GetName() + "; "
	}
	addCnt, addMsg := p.unassigningPool.AddPods(errPods)
	logger.Info("rollback error pods", zap.String("msg", msg), zap.String("add UnassigningPool msg", addMsg),
		zap.Any("len", len(errPods)), zap.Any("addCnt for UnassigningPool", addCnt), zap.String("pod", podsStr))
}

// jobs:
//  1. scan SharePool/VIPPool:
//  1. Running/Sharing: nop
//  2. ReadyUnassign: move to UnassigningPool, trigger grpc to unassign.
//  3. Unassigning: log error, should be in UnassigningPool.
//  4. Unassigned: log error, should be in UnassignedPool/WarmupPool.
//  5. Pending: log error, should be in WarmupPool
//  6. Error: change to ReadyUnassign, move to UnassigningPool, trigger grpc to unassign
//  7. Delete: just delete from pool.
//  2. scan UnassigningPool:
//  1. Running/Sharing: log error, should be in SharePool/VIPPool
//  2. ReadyUnassign: log error, should be in SharePool/VIPPool
//  3. Unassigning: nop
//  4. Unassigned: move to WarmupPool.
//  5. Pending: log error, should be in WarmupPool
//  6. Error: trigger rpc to unassign, change to Unassigning
//  7. Delete: just delete from pool.
//  3. scan WarmupPool:
//  1. Running/Sharing: log error, should be in SharePool/VIPPool
//  2. ReadyUnassign: log error, should be in SharePool/VIPPool
//  3. Unassigning: log error, should be in UnassigningPool
//  4. Unassigned: nop
//  5. Pending: nop
//  6. Error: log error, should be in UnassigningPool
//  7. Delete: just delete from pool.
//
// 4. handle lostPods
func (p *PodMgr) podScannerLoop() {
	logger := p.logger.With(zap.String(logger.LogIDBgWorker, "podScannerLoop"))
	for {
		start := time.Now()
		curConfig := p.configMgr.GetConfig()
		warmupPoolCap := *curConfig.GetClusterConfig().WarmupPoolCap

		// scan SharePool and VIPPool to get all pods that will be moved to UnassigningPool
		// (ReadyUnassign/LogicalError)
		podsToUnassign, podsToWarmupPool := p.sharePool.scanUsablePool(logger)
		podsToUnassign1, podsToWarmupPool1 := p.vipPool.scanUsablePool(logger)
		for podName, podDesc := range podsToUnassign1 {
			if _, ok := podsToUnassign[podName]; ok {
				logger.Error("got pod in VIPPool which is duplicated to pod in SharePool, ignore",
					zap.String("pod", podName), zap.String("status", podDesc.GetStatusStr()))
			} else {
				podsToUnassign[podName] = podDesc
			}
		}

		p.unassigningPool.AddAndTriggerUnassign(podsToUnassign, logger)
		podsToWarmupPool2 := p.unassigningPool.scanPool(logger)
		for podName, podDesc := range podsToWarmupPool1 {
			if _, ok := podsToWarmupPool[podName]; ok {
				logger.Error("got pod in VIPPool which is duplicated to pod in SharePool, ignore",
					zap.String("pod", podName), zap.String("status", podDesc.GetStatusStr()))
			} else {
				podsToWarmupPool[podName] = podDesc
			}
		}
		for _, podDesc := range podsToWarmupPool2 {
			podName := podDesc.GetName()
			if _, ok := podsToWarmupPool[podName]; ok {
				logger.Error("got pod in UnassigningPool which is duplicated to pod in SharePool, ignore",
					zap.String("pod", podName), zap.String("status", podDesc.GetStatusStr()))
			} else {
				podsToWarmupPool[podName] = podDesc
			}
		}
		podsToWarmupPoolFinal := make([]*meta.PodDesc, 0, len(podsToWarmupPool))
		for _, podDesc := range podsToWarmupPool {
			podsToWarmupPoolFinal = append(podsToWarmupPoolFinal, podDesc)
		}

		addCnt, dupMsg := p.warmupPool.AddPods(podsToWarmupPoolFinal)
		if len(dupMsg) != 0 {
			logger.Error("add pods to WarmupPool got duplicated pods",
				zap.Any("add cnt", addCnt), zap.Any("real", len(podsToWarmupPool)), zap.Any("dup pods", dupMsg))
		}

		podsToDelete := p.warmupPool.scanPool(logger, *curConfig.GetClusterConfig().PendingPodTimeoutSec)
		// No need to change cloneset replicas, only need to remove error pod,
		// and new pod will be created by cloneset automatically.
		if err := p.k8sMgr.DelK8sPods(podsToDelete, warmupPoolCap, 3, false); err != nil {
			logger.Error("delete pods from WarmupPool failed", zap.Any("pods", podsToDelete), zap.Error(err))
		}

		p.handleLostPodsLoop(curConfig)

		p.logger.Info("pod mgr scanner iteration done", zap.Any("elapsed", time.Since(start)))

		scannerIntervalSec := *curConfig.GetClusterConfig().PodScannerIntervalSec
		select {
		case <-time.After(time.Duration(scannerIntervalSec) * time.Second):
		case <-p.ctx.Done():
			p.logger.Info("got ctx done, exit")
		}
	}
}

// handleLostPodsLoop is used for crash recovery of AutoScaler.
// If AutoScaler crashed and lost all meta info, this func will list all k8s pods
// and check if there are corresponding meta info. If not for a long time,
// this pod is treated a lost pod and will be delete.
// called by PodMgr.podScannerLoop
func (p *PodMgr) handleLostPodsLoop(curConfig *config.Config) {
	logger := p.logger.With(zap.String(logger.LogIDBgWorker, "handleLostPodsLoop"))

	start := time.Now()
	lostWindowSec := *curConfig.GetClusterConfig().LostPodWindowSec
	warmupPoolCap := *curConfig.GetClusterConfig().WarmupPoolCap

	// collect lost pods.
	allPods, _ := p.GetAllPodsMap()
	// todo log k8sPods
	_, k8sPods, err := p.k8sMgr.ListPods()
	if err != nil {
		p.logger.Error("list pods failed", zap.Error(err))
	} else {
		k8sPodsStr := printK8sPods(k8sPods)
		p.logger.Info("list pods info", zap.Any("info", k8sPodsStr))

		for podName := range k8sPods {
			if _, ok := allPods[podName]; !ok {
				if _, ok1 := p.lostPods[podName]; !ok1 {
					logger.Info("got lost candidate pod", zap.String("pod", podName), zap.Any("lostTs", start))
					p.lostPods[podName] = start
				}
			}
		}
	}

	podsToDel := make([]string, 0, len(p.lostPods))
	for podName, lostTs := range p.lostPods {
		if _, ok := allPods[podName]; ok {
			delete(p.lostPods, podName)
			continue
		}
		if start.After(lostTs.Add(time.Duration(lostWindowSec) * time.Second)) {
			logger.Info("got lost pod", zap.String("pod", podName), zap.Any("lostTs", lostTs),
				zap.Any("now", start), zap.Any("lost window", lostWindowSec))
			podsToDel = append(podsToDel, podName)
			delete(p.lostPods, podName)
		}
	}
	if len(podsToDel) > 0 {
		logger.Info("deleting lost pods", zap.Any("lost", podsToDel))
		p.k8sMgr.DelK8sPods(podsToDel, warmupPoolCap, 3, true)
	}

	logger.Info("handleLostPodsLoop done", zap.Any("elapsed", time.Since(start)),
		zap.Any("lostPods", p.lostPods))
}

func (p *PodMgr) DumpAllPods() string {
	_, allPods := p.GetAllPodsMap()
	resStr := "share:"
	podsMap := allPods[0]
	resStr += sortAndStringfy(podsMap)

	resStr += " vip:"
	podsMap = allPods[1]
	resStr += sortAndStringfy(podsMap)

	resStr += " unassign:"
	podsMap = allPods[2]
	resStr += sortAndStringfy(podsMap)

	resStr += " warmup:"
	podsMap = allPods[3]
	resStr += sortAndStringfy(podsMap)

	return resStr
}

// todo when del pod, always try delete non running pod first.
func (p *PodMgr) warmupLoop() {
	logger := p.logger.With(zap.String(logger.LogIDBgWorker, "warmupLoop"))
	var retryCnt uint32 = 3
	lastDumpPodsTs := time.Now()
	lastForceOnePodTs := time.Now()
	for {
		// TODO maybe conflict with original cloneset definition.
		start := time.Now()
		clusterConfig := p.configMgr.GetConfig().GetClusterConfig()
		warmupPoolCap := *clusterConfig.WarmupPoolCap
		poolCap := *clusterConfig.WarmupPoolCap
		warmupIntervalSec := *clusterConfig.WarmupIntervalSec
		filter := func(pod *meta.PodDesc) bool {
			return !pod.IsDeleted()
		}
		curSize := p.warmupPool.GetPodsNum(filter)
		failCnt := uint(p.failCnt.Swap(0))
		expectSize := poolCap + failCnt
		delta := int(expectSize) - int(curSize)

		logger.Info("warmup info, start to call k8s API", zap.Any("poolCap", poolCap), zap.Any("failCnt", failCnt),
			zap.Any("curSize", curSize), zap.Any("delta", delta),
			zap.Any("warmupIntervalSec", warmupIntervalSec))

		if start.After(lastDumpPodsTs.Add(time.Duration(10) * time.Second)) {
			lastDumpPodsTs = time.Now()
			logger.Debug("warmup info, dump pods", zap.Any("dump all pods", p.DumpAllPods()))
		}

		if delta < 0 {
			delNum := uint(-delta)

			// Remove pods first time, which will only remove non-ready pods.
			delPods1 := p.warmupPool.RemovePodsByNum(delNum, func(pod *meta.PodDesc) bool {
				return !pod.IsDeleted() && !pod.IsUnassigned()
			})

			finalDelPods := delPods1
			if uint(len(delPods1)) < delNum {
				// Remove pods again, which may remove ready pods.
				delPods2 := p.warmupPool.RemovePodsByNum(delNum-uint(len(delPods1)), filter)

				finalDelPods = make([]*meta.PodDesc, 0, len(delPods1)+len(delPods2))
				finalDelPods = append(finalDelPods, delPods1...)

				// Make sure `delPods1` and `delPods2` doesn't overlap.
				for _, podDesc2 := range delPods2 {
					var samePod bool
					for _, podDesc1 := range delPods1 {
						if podDesc1.GetName() == podDesc2.GetName() {
							samePod = true
							break
						}
					}

					if samePod {
						logger.Error("warmup info, got same pod when remove the second time", zap.Any("pod", podDesc2.GetName()))
					} else {
						finalDelPods = append(finalDelPods, podDesc2)
					}
				}
			}

			delPodsStr := make([]string, 0, len(finalDelPods))
			for _, podDesc := range finalDelPods {
				delPodsStr = append(delPodsStr, podDesc.GetName())
			}
			if err := p.k8sMgr.DelK8sPods(delPodsStr, warmupPoolCap, retryCnt, true); err != nil {
				logger.Error("warmup info, p.k8sMgr.DelK8sPods failed", zap.Error(err))
			}
		} else if delta > 0 {
			if err := p.checkAndAddK8sPods(uint(delta), retryCnt, clusterConfig, logger); err != nil {
				logger.Error("warmup info, p.checkAndAddK8sPods failed", zap.Error(err))
			}
		}

		readyNum := p.warmupPool.GetPodsNum(func(pod *meta.PodDesc) bool {
			return pod.IsUnassigned()
		})
		if readyNum <= 0 {
			now := time.Now()
			if now.After(lastForceOnePodTs.Add(time.Duration(*clusterConfig.WarmupPoolOnePodCoolDownSec) * time.Second)) {
				logger.Info("warmup info, ready pods is zero, force add one pod")

				if err := p.checkAndAddK8sPods(1, retryCnt, clusterConfig, logger); err != nil {
					logger.Error("warmup info, p.checkAndAddK8sPods failed", zap.Error(err))
				}
				lastForceOnePodTs = now
			} else {
				logger.Info("warmup info, ready pods is zero, but in cooldown duration")
			}
		}

		logger.Info("warmup info", zap.Any("elapsed", time.Since(start)))
		select {
		case <-time.After(time.Duration(warmupIntervalSec) * time.Second):
		case <-p.ctx.Done():
			logger.Info("got ctx.Done(), exit")
			return
		}
	}
}

func (p *PodMgr) checkAndAddK8sPods(num uint, retryCnt uint32,
	clusterConfig *config.ClusterConfig, logger *zap.Logger) error {
	maxPods := *clusterConfig.ClusterMaxPods

	if maxPods <= 0 {
		return p.k8sMgr.AddK8sPods(num, retryCnt)
	}

	if allPods, _ := p.GetAllPodsMap(); len(allPods) >= maxPods {
		logger.Info("cur pods exceeds maxPods, skip adding new pods",
			zap.Any("curPods", len(allPods)), zap.Any("maxPods", maxPods))
		return nil
	} else if len(allPods)+int(num) > maxPods {
		newNum := uint(maxPods - len(allPods))
		logger.Info("cur pods will exceed maxPods after add new ones, changing delta",
			zap.Any("maxPods", maxPods), zap.Any("curPods", len(allPods)), zap.Any("oldNum", num),
			zap.Any("newNum", newNum))
		num = newNum
	}

	return p.k8sMgr.AddK8sPods(num, retryCnt)
}

func (p *PodMgr) takePodsFromSharePool(num uint, tenantName string, curConfig *config.Config, logger *zap.Logger) []*meta.PodDesc {
	clusterConfig := curConfig.GetClusterConfig()
	tenantConfig := curConfig.GetTenantConfig(tenantName)
	sharePoolConfig := clusterConfig.SharePoolConfig
	return p.sharePool.TryTake(tenantName, num,
		*sharePoolConfig.ShareLowCPULimit,
		*sharePoolConfig.ShareCoolDownSec,
		*clusterConfig.MetricWindowSec,
		*clusterConfig.MetricIntervalSec,
		*clusterConfig.CoreNumPerPod,
		*tenantConfig.StillShareWhenMetricNotEnough,
		logger)
}

func (p *PodMgr) GetAllPodsMap() (meta.PodsMap, []meta.PodsMap) {
	p.rlockAllPools()
	defer p.runlockAllPools()

	pods1 := p.sharePool.GetPodsMapWithoutLock(nil)
	pods2 := p.vipPool.GetPodsMapWithoutLock(nil)
	pods3 := p.unassigningPool.GetPodsMapWithoutLock(nil)
	pods4 := p.warmupPool.GetPodsMapWithoutLock(nil)

	res := make(meta.PodsMap, len(pods1)+len(pods2)+len(pods3)+len(pods4))
	allPods := []meta.PodsMap{pods1, pods2, pods3, pods4}
	for _, podsMap := range allPods {
		for podName, podDesc := range podsMap {
			if _, ok := res[podName]; ok {
				p.logger.Error("got duplicated pod when GetAllPodsMap", zap.String("dup pod", podName))
			} else {
				res[podName] = podDesc
			}
		}
	}
	return res, allPods
}

func (p *PodMgr) rlockAllPools() {
	p.sharePool.basePool.mu.RLock()
	p.vipPool.basePool.mu.RLock()
	p.warmupPool.basePool.mu.RLock()
	p.unassigningPool.basePool.mu.RLock()
}

func (p *PodMgr) runlockAllPools() {
	p.sharePool.basePool.mu.RUnlock()
	p.vipPool.basePool.mu.RUnlock()
	p.warmupPool.basePool.mu.RUnlock()
	p.unassigningPool.basePool.mu.RUnlock()
}

func (p *PodMgr) AddPodDesc(k8sPod *k8sapiv1.Pod) {
	phase := k8sPod.Status.Phase
	var (
		isPending bool
		msg       string
		addCnt    uint
		dupMsg    string
		pod       *meta.PodDesc
	)
	defer func() {
		p.logger.Info("AddPodDesc done", zap.Any("pod", k8sPod.Name), zap.Any("msg", msg))
	}()

	if phase == k8sapiv1.PodPending {
		msg = "pod is pending"
		isPending = true
	} else if phase == k8sapiv1.PodRunning {
	} else {
		// handleLostPodsLoop will delete it later.
		// msg = fmt.Sprintf("pod is not pending or running, ignore adding")
		// return
		msg = fmt.Sprintf("pod is %v", phase)
		isPending = true
	}

	// Check if pods already exists in pools to avoid situation like:
	// 1. pod in VIPPool changed to Pending(maybe evicted by k8s)
	// 2. pod recovery by k8s and AddPodDesc is called before warmup loop cleaning Pending pod.
	pods := p.warmupPool.GetPodsByName([]string{k8sPod.Name})
	if len(pods) == 1 {
		pod = pods[0]
	}
	var ok bool
	if pod == nil {
		allPods, _ := p.GetAllPodsMap()
		pod, ok = allPods[k8sPod.Name]
	}

	if ok || pod != nil {
		msg = fmt.Sprintf("pod(%s) already in some pool, skip adding", k8sPod.Name)
	} else {
		pod = meta.NewPodDesc(k8sPod.Name, k8sPod.Status.PodIP, namespace, isPending)
		addCnt, dupMsg = p.warmupPool.AddPods([]*meta.PodDesc{pod})
		msg = fmt.Sprintf("addCnt: %v, dupMsg: %v", addCnt, dupMsg)
		if addCnt == 1 && phase == k8sapiv1.PodRunning {
			p.warmupPool.NotifyNewWarmupPod()
		}
	}
}

func (p *PodMgr) DelPodDesc(k8sPod *k8sapiv1.Pod) {
	allPods, _ := p.GetAllPodsMap()
	if podDesc, ok := allPods[k8sPod.Name]; ok {
		p.logger.Info("tag pod status to deleted", zap.Any("pod", k8sPod.Name))
		podDesc.ToDeleted()
	} else {
		p.logger.Info("cannot find podDesc to delete", zap.Any("pod", k8sPod.Name))
	}
}

func (p *PodMgr) UpdatePodDesc(k8sPod *k8sapiv1.Pod) {
	var (
		pod *meta.PodDesc
		msg string
	)
	defer func() {
		p.logger.Info("UpdatePodDesc", zap.Any("pod", k8sPod.Name), zap.Any("msg", msg))
	}()

	// In most situation, pending pods are in WarmupPool, so we check WarmupPool first
	pods := p.warmupPool.GetPodsByName([]string{k8sPod.Name})
	if len(pods) == 1 {
		pod = pods[0]
	}
	if pod == nil {
		allPods, _ := p.GetAllPodsMap()
		var ok bool
		if pod, ok = allPods[k8sPod.Name]; !ok {
			// Why can't the unfound pod be re-added to the WarmupPool:
			// Because after calling the CloneSet delete API,
			// Kubernetes may return an event notification of a pod update,
			// but at that time, the pod status may still be Running. If it is re-added, it will cause the deletion to fail.
			// TODO: some pod may cannot be found while it has already been take out from Pool-A,
			// and not yet to be put into Pool-B. In this case, we lost this update. Need to fix later.
			msg = fmt.Sprintf("cannot find pod to update in all pools, phase: %v", k8sPod.Status.Phase)
			return
		}
	}

	// in dev, pod got here because eviction or deleted manually.
	// if k8sPod.DeletionTimestamp != nil || k8sPod.DeletionGracePeriodSeconds != nil {
	// 	if k8sPod.DeletionTimestamp != nil {
	// 		msg += fmt.Sprintf("got DeletiongTimesamp: %v. ", *k8sPod.DeletionTimestamp)
	// 	}
	// 	if k8sPod.DeletionGracePeriodSeconds != nil {
	// 		msg += fmt.Sprintf("got DeletionGracePeriodSeconds: %v", *k8sPod.DeletionGracePeriodSeconds)
	// 	}
	// 	msg += "; pod change to LogicalError"
	// 	pod.ToLogicalError()
	// 	return
	// }

	// only cares PodIP and Phase.
	if k8sPod.Status.Phase == k8sapiv1.PodRunning {
		// When got following pod condition, it means pod will be evicted.
		// We need to mark it as Pending as soon as possible.
		// And it will be updated to running status later.
		//   {"type":"DisruptionTarget","status":"True","lastProbeTime":null,
		//    "lastTransitionTime":"2024-11-06T06:12:35Z","reason":"EvictionByEvictionAPI","message":"Eviction API: evicting"}],
		podConditions := k8sPod.Status.Conditions
		condLen := len(podConditions)
		if condLen != 0 {
			lastCondition := podConditions[condLen-1]
			// TODO: this may change because upgrade k8sapi version!!
			if lastCondition.Type == k8sapiv1.DisruptionTarget {
				pod.ToPending()
				msg = fmt.Sprintf("k8sPod status is not ok, it will be evicted (phase: %v, cond: %v), change pod status to Pending",
					k8sPod.Status.Phase, lastCondition.Reason)
				return
			}
		}

		statusChanged := pod.ChangeIPAndFromPendingToUnassigned(k8sPod.Status.PodIP)
		msg = fmt.Sprintf("change pod ip and update pod status, pod: %v, changed: %v",
			pod.ToString(), statusChanged)
		if pod.IsUnassigned() {
			p.warmupPool.NotifyNewWarmupPod()
		}
	} else if k8sPod.Status.Phase == k8sapiv1.PodPending {
		pod.ToPending()
		msg = fmt.Sprintf("k8sPod status is pending, change pod status to Pending")
	} else {
		pod.ToLogicalError()
		msg = fmt.Sprintf("k8sPod status is not ok(%v), change pod status to Error", k8sPod.Status.Phase)
	}
}

// UpdateMetricsByCopy update metrics of all pods of SharePool and VIPPool.
func (p *PodMgr) UpdateMetricsByCopy(
	cpu metric.PodsCPUMetricWindow,
	oom metric.PodsOOMMetricWindow,
	taskCnt metric.PodsTaskCntMetricWindow,
	ru metric.PodsRUMetricWindow) {
	updater := func(pods meta.PodsMap) {
		for podName, podDesc := range pods {
			cpuWindowCopy := metric.CPUMetricWindow{}
			taskCntWindowCopy := metric.TaskCntMetricWindow{}
			oomWindowCopy := metric.OOMMetricWindow{}
			ruWindowCopy := metric.RUMetricWindow{}

			if window, ok := cpu[podName]; ok {
				cpuWindowCopy = window.DeepCopy()
			}

			if window, ok := taskCnt[podName]; ok {
				taskCntWindowCopy = window.DeepCopy()
			}

			if window, ok := oom[podName]; ok {
				oomWindowCopy = window.DeepCopy()
			}

			if window, ok := ru[podName]; ok {
				ruWindowCopy = window.DeepCopy()
			}

			podDesc.UpdateMetrics(cpuWindowCopy, taskCntWindowCopy, oomWindowCopy, ruWindowCopy)
		}
	}
	updater(p.sharePool.GetPodsCopy())
	updater(p.vipPool.GetPodsCopy())
}

func sortAndStringfy(podsMap meta.PodsMap) string {
	s := make([]T, 0, len(podsMap))
	for podName, podDesc := range podsMap {
		s = append(s, T{
			name: podName,
			pod:  podDesc,
		})
	}
	sort.Slice(s, func(i, j int) bool {
		return s[i].name < s[j].name
	})
	var resStr string
	for _, t := range s {
		resStr += t.pod.ToString()
	}
	return resStr
}