#!/bin/env bash
##Description: standby start replicate

script_run_role="PRIMARY,STANDBY"
script_base_name="tidb_so_standby_startrepl"

function f_syntax
{
    [ ! -z "$1" ] && echo ">>[$(date "+%Y%m%d.%H%M%S")] $1"
    echo " "
    echo "bash ${ShellName}  [-primaryhostname <hostname>]"
    echo "-standbyhostname <hostname_standby> "
    echo " "
    echo " #Description:"
    echo " -primaryhostname ,the primary cluster manager host"
    echo " -standbyhostname ,the standby cluster manager host"
    echo " "
    echo " #sample:"
    echo " #sh ${ShellName} -primaryhostname node1 -standbyhostname sa-172-16-4-81" 
    echo " "
    exit 22
}

function f_PrintLog
{
    _LogFlag=$(echo "$1"|tr "a-z" "A-Z")
    _LogMessage=$2
    [ $(echo "$1"|wc -L) -eq 4 ] && _LogFlag=$(echo "${_LogFlag}_")
    echo "$(date "+%Y%m%d.%H%M%S"):[${_LogFlag}]:${UserName}@${HostName}:${2}";
}

###############################################################################



function f_runscp
{
    _file_source=$1
    _file_tartget=$2
    _nodid=$3
    _action=$4
    f_PrintLog "Info" "Start to ${_action} file:${_file_source} ${_file_tartget}, remote:${_nodid}"
    echo "tiup cluster ${_action} ${clsName} ${_file_source} ${_file_tartget} -N${_nodid}"
    tiup cluster ${_action} ${clsName} ${_file_source} ${_file_tartget} -N${_nodid}
    if [ $? -ne 0 ]
    then
        f_PrintLog "Error" "tiup scp file failed, remote host:${_nodid}, file:${_file_source}"
        exit 1
    else
        f_PrintLog "Info" "tiup scp file finish, remote host:${_nodid}, file:${_file_source}"
    fi
}

function f_runsqlstmt2
{
    _execsql=$1
    f_PrintLog "Info" "START TO RUN SQL STMT:${_execsql}, DBinfo:${connOk}"
    exec_result=$(mysql --login-path=${locallogin:-tidblogin} -NBe "${_execsql}")
    _rt_code=$?
    if [ $_rt_code -ne 0 ]
    then
        f_PrintLog "Error" "Run stmt failed, exit"
        exit 1
    fi
}



function f_runsqlstmt1
{
    _execsql=$1
    f_PrintLog "Info" "START TO RUN SQL STMT:${_execsql}, DBinfo:${connOk}"
    mysql --login-path=${locallogin:-tidblogin} -vvv -e "${_execsql}"
    _rt_code=$?
    if [ $_rt_code -ne 0 ]
    then
        f_PrintLog "Error" "Run stmt failed, exit"
        # exit 1
    fi
}


function f_runsqlstmt
{
    _ip=$1
    _execsql=$2

    #主备区分执行
    if [ "${_ip}" = "${tidbIpLocal}" ]
    then 
        connstr="--login-path=${locallogin:-tidblogin}"
    else
        connstr="-h${_ip} -P${tidbPort} -u${user} -p${tidbPasswd:-123}"
    fi

    f_PrintLog "Info" "START TO RUN SQL STMT:${_execsql}, DBinfo ${_ip}:${_port}"
    mysql ${connstr} -vvv -e "${_execsql}"
    return $?
}



function f_runsqlfile
{
    _sqlfile=$1
    _ip=$2

    #主备区分执行
    if [ "${_ip}" = "${tidbIpLocal}" ]
    then 
        connstr="--login-path=${locallogin:-tidblogin}"
    else
        connstr="-h${_ip} -P${tidbPort} -u${user} -p${tidbPasswd:-123}"
    fi

    f_PrintLog "Info" "START TO RUN SQL:${_sqlfile}, DBinfo ${_ip}:${_port}"
    echo "*************************************************************"
    cat ${_sqlfile}
    echo "*************************************************************"
    f_PrintLog "Info" "PLS CHECK ALL COMMANDS IN FILE, SLEEP 3......"
    sleep 3
    echo "mysql ${connstr} -vvv < ${_sqlfile}"
    mysql ${connstr} -vvv < ${_sqlfile}
    return $?
}

function f_tiupcmd
{
    f_PrintLog "Info" "Start to run f_tiupcmd."
    cls_name="$1"
    component_name="$2"
    action="$3"
    
    f_PrintLog "Info" "Start to ${action} ${component_name} on cluster ${cls_name} ..."
    sleep 3
    #clsinfo 
    tiup cluster ${action} ${cls_name} -N${component_name} --yes --wait-timeout 60
    rt_code=$?
    if [ ${rt_code} -eq 0 ]
    then
        f_PrintLog "Info" "${action} ${component_name} on cluster ${cls_name} successfully!"
        return ${rt_code}
    else
        f_PrintLog "Error" "${action} ${component_name} on cluster ${cls_name} failed!!!"
        return ${rt_code}
    fi
}

function f_committs_stop
{
    _ip_s=$1

    f_PrintLog "Info" "Start to run f_committs_stop."

    #最多检查3次,每次间隔10s
    for i in 1 2 3 
    do

        #commit ts0
        ck_sql="select checkPoint from ${checkpointschema}.checkpoint order by clusterID desc limit 1"
        f_PrintLog "Info" "START TO RUN SQL:${ck_sql}, DBinfo ${_ip_s}"
        exec_result=$(mysql -h${_ip_s} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${ck_sql}")
        if [ $? -ne 0 ]
        then
            f_PrintLog "Error" "run check sql failed, STMT:${ck_sql}, pls check!!!"
            exit 1
        fi

        cmt_tso=$(echo $exec_result  | python -m json.tool | awk '{
        if($1~"commitTS"){gsub(",","",$2);printf "%s\n",$2;exit}
        }')
        consistent=$(echo $exec_result  | python -m json.tool | awk '{
        if($1~"consistent"){gsub(",","",$2);printf "%s\n",$2;exit}
        }')
        echo $exec_result  | python -m json.tool
        
        ck_sql="select from_unixtime(conv(left(bin('${cmt_tso}'),41),2,10)/1000) "
        cmt_ts=$(mysql -h${_ip_s} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${ck_sql}")
        
        #time-tso diff
        ck_sql="SELECT timestampdiff(SECOND,TIDB_PARSE_TSO(${cmt_tso}),NOW()) "
        exec_result=$(mysql -h${_ip_s} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${ck_sql}")
        f_PrintLog "Info" "Drainer checkpoint consistent is ${consistent} (should be true), commitTS is ${cmt_tso}, ${cmt_ts}, is ${exec_result} (should>60) seconds away from now!"
        
        [ ${exec_result} -gt 60 -o "${consistent}" = "true" ] && return 0
        sleep 10
    done 
    f_PrintLog "Error" "Drainer checkpoint check failed, drainer maybe in running, pls check!!!"
    exit 1
}

function f_grant_user
{
    _ip=$1
    _grantfile=$2 

    f_PrintLog "Info" "Start to run f_grant_user"
    if [ ! -f "${_grantfile}" ]
    then
        f_PrintLog "Error" "User grant backup file not exits, pls check ${_grantfile}, quit!!!"
        exit 1
    fi

    f_runsqlfile "${_grantfile}" "${_ip}" 
    _rt_code=$?
    if [ ${_rt_code} -eq 0 ]
    then
        f_PrintLog "Info" "User grant backup file exec finish!"
    else
        f_PrintLog "Error" "User grant backup file exec failed!!!"
        # exit 1
    fi

}

function f_drop_checkpoint
{
    _ip=$1
    f_PrintLog "Info" "Start to run f_drop_checkpoint."

    #sql
    drop_tab="drop table ${checkpointschema}.checkpoint"
    select_tab="select * from ${checkpointschema}.checkpoint"
    
    #Primary
    f_runsqlstmt "${_ip}" "${select_tab}"
    f_runsqlstmt "${_ip}" "${drop_tab}"
    if [ $? -ne 0 ]
    then
        f_PrintLog "Error" "exec sql failed, stmt:${drop_tab}"
        exit 1
    fi
}

function f_checktidbbinlog
{
    binlog_url=$1

    f_PrintLog "Info" "Start to run f_checktidbbinlog"
    # binlogstatus=$(curl "http://${binlog_url}/info/all")
    # f_formatjson ${binlogstatus}
    binlog_down=$(curl -s "http://${binlog_url}/info/all" | awk -F: '{
        if($1~"binlog_status" && $2!~"On"){gsub("[, ]","",$2);gsub("\"","",$2);printf "%s",$2;exit}
    }')

    binlog_up=$(curl -s "http://${binlog_url}/info/all" | awk -F: '{
        if($1~"binlog_status" && $2~"On"){gsub("[, ]","",$2);gsub("\"","",$2);printf "%s",$2;exit}
    }')
    
    curl -s "http://${binlog_url}/info/all" | awk -F: '
    /^[ "]*ip.*/{gsub("[, ]","",$2);gsub("\"","",$2);ip=$2}
    /^[ "]*listening_port.*/{gsub("[, ]","",$2);gsub("\"","",$2);port=$2}
    /^[ "]*binlog_status.*/{gsub("[, ]","",$2);gsub("\"","",$2);binlog=$2}
    /^[ "]*server_id.*/{TiDB[$2]=tidb;IP[$2]=ip;Port[$2]=port;Binlog[$2]=binlog;}
    END{for(a in TiDB){printf "binlog_status: %s:%s %s\n",IP[a],Port[a],Binlog[a]}}
    '

    if [ "${binlog_down:-On}" = "On" -a "${binlog_up:-Off}" = "On" ]
    then
        f_PrintLog "Info" "binlog_status is ok."
    else
        f_PrintLog "Error" "binlog_status check failed. binlog_status is ${binlog_up}:${binlog_down}"
        exit 1
    fi
}

function f_checkbinlog
{
    _clsname=$1

    f_PrintLog "Info" "Start to run f_checkbinlog"
   #check binlog status
    tidburl=$(tiup cluster display ${_clsname} 2>/dev/null |awk '{
        if(NF>=8 && $2=="tidb" && $6=="Up" ){gsub(".*/","",$4);printf "%s:%s\n",$3,$4;exit}
        }')
    if [ -z "${tidburl}" ] 
    then
        f_PrintLog "Error" "Failed to check tidb binlog status. tidburl is null!!!"
        exit 1
    else
        f_PrintLog "Info" "Start to check tidb binlog status. tidburl: http://${tidburl}/info/all "
        f_checktidbbinlog ${tidburl}
    fi
}

function f_sacleoutdrainer
{
    _clsname=$1
    _scaleyaml=$2
    f_PrintLog "Info" "Start to scale cluster ${_clsname}, config file:${_scaleyaml}"
    echo "************************"
    cat ${_scaleyaml}
    echo "************************"
    sleep 3
    _nd_true=$(awk -F: '{if($1~"ignore_exporter"){gsub(" ","",$2);printf "%s",$2}}' ${_scaleyaml} 2>/dev/null )
    if [ "${_nd_true}" != "true" ]
    then
        f_PrintLog "Warn" "config file not contain ignore_exporter=true, pls check yaml file: ${_scaleyaml}"
        exit 1
    fi
    
    ssh_kdir="${TidbHome}/.tiup/storage/cluster/clusters/${_clsname}/ssh/id_rsa"
    if [ -f ${ssh_kdir} ]
    then
        ssh_kstr="-i ${ssh_kdir}"
        f_PrintLog "Info" "Scale-out with ssh key: ${ssh_kdir}"
    fi

    tiup cluster scale-out ${_clsname} ${_scaleyaml}  --yes --wait-timeout 60 
    _rt_code=$?

    tiup cluster display ${_clsname}
    newDrainerId=$(tiup cluster display ${_clsname} 2>/dev/null|awk -v id="${tidbIp}" '{
        if($1~id && $2=="drainer" && $6=="Up"){printf "%s",$1;exit}
        }')

    if [ ${_rt_code} -eq 0 -a -n "${newDrainerId}" ]
    then
        f_PrintLog "Info" "tiup cluster scale-out finish, drainer id:${newDrainerId}"
    else
        f_PrintLog "Error" "tiup cluster scale-out fail, pls check logs!!!"
        exit 1
    fi    
}

function f_startdrainer
{
    _clsname=$1
    _tidbDrainerId=$2

    f_PrintLog "Info" "Start to run f_startdrainer."
    #check tidb
    tiup cluster display ${_clsname}
    if [ -z "${_tidbDrainerId}" ] 
    then
        f_PrintLog "Error" "Can not find valid drainer node ${_tidbDrainerId}, the cluster name is ${_clsname}"
        exit 1
    fi

    #drainer ip handle
    tidbclsinfo=$(tiup cluster display ${_clsname}  2>/dev/null)

    #启动停止的drainer
    drainerStatus=$(echo "${tidbclsinfo}"|awk -v id="${_tidbDrainerId}" '{
        if($1~id && $2=="drainer"){printf "%s",$6;exit}
        }')

    drainerID=$(echo "${tidbclsinfo}"|awk -v id="${_tidbDrainerId}" '{
        if($1~id && $2=="drainer"){printf "%s",$1;exit}
        }')

    if [ "${drainerStatus}" = "Up" ]
    then
        f_PrintLog "Warn" "Drainer ${_tidbDrainerId} status is ${drainerStatus}, not need to start." 
        return 0   
    elif [ "${drainerStatus}" = "Down" ]
    then
        action="start"
        f_tiupcmd "${_clsname}" "${drainerID}" "${action}"
        rt_code=$?
    else
        f_PrintLog "Error" "Drainer ${_tidbDrainerId} status is ${drainerStatus}, exit!!!"
        exit 1
    fi


    tiup cluster display ${_clsname} 
    tidbclsinfo=$(tiup cluster display ${_clsname} 2>/dev/null)
    drok=$(echo "${tidbclsinfo}"|awk -v dip="${drainerID}" '{
        if($1==dip && $2=="drainer" && $6=="Up"){printf "%s",$1;exit}
        }')

    if [ ${rt_code} -eq 0 -a -n "${drok}" ]
    then
        f_PrintLog "Info" "cluster ${_clsname} node ${drainerID} ${action} finish!"    
    else
        f_PrintLog "Error" "cluster ${_clsname} node ${drainerID} ${action} failed !!!"
        exit 1
    fi

}


function f_check_insert
{
    _ip=$1
    _ip_s=$2

    #insert into primary, check standby replicate
    f_PrintLog "Info" "Start to run f_check_insert."
    f_PrintLog "Info" "Start to insert ${_ip} -> ${_ip_s}."

    #sql
    tmp_table="test.drainer_delay"
    create_tab="create table If Not Exists ${tmp_table}(ts datetime)"
    delete_tab="delete from ${tmp_table}"
    insert_tab="insert into ${tmp_table}(ts) values(now())"
    select_tab="select max(ts) from ${tmp_table}"
    #
    
    #Primary
    # f_runsqlstmt "${_ip}" "${tidbPort}" "${user}" "${tidbPasswd:-123}"  "${create_tab}"
    f_runsqlstmt "${_ip}" "${delete_tab}"
    f_runsqlstmt "${_ip}" "${insert_tab}"
    f_runsqlstmt "${_ip}" "${select_tab}"

    
    s_val=$(mysql -h${_ip} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${select_tab}")
    if [ $? -ne 0 ]
    then
        f_PrintLog "Error" "exec sql failed, stmt: ${select_tab}"
    fi
    
    sleep 1
    #Standby
    # f_PrintLog "Info" "START TO RUN SQL STMT:${select_tab}, DBinfo ${s_ip}:${s_port} "
    f_runsqlstmt "${_ip_s}" "${select_tab}"
    for i in 1 2 3
    do
        f_runsqlstmt "${_ip_s}" "${select_tab}"
        p_val=$(mysql -h${_ip_s} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${select_tab}")
        if [ "${s_val:-0}" = "${p_val:-1}" ]
        then
            # break
            f_PrintLog "Info" "Real time Data replicate successful: ${p_val}=${s_val}, check pass! "
            f_PrintLog "Succ" "Tmp table insert and replicate successful, drainer working good!"
            return 0
        fi
        f_PrintLog "Warn" "Real time Data replicate failed: ${p_val}!=${s_val}, continue ${i} !!! "
        sleep 3
    done

    f_PrintLog "Error" "Real time data replicate:${p_val}!=${s_val}, check failed!!!"
    f_PrintLog "Error" "Tmp table insert and replicate failed, pls check data in ${tmp_table} !!!"
    exit 1
}

function f_backfile
{
    _filename=$1
    _filepath="${_filename%/*}/baktmp"
    [ ! -d ${_filepath} ] && mkdir -p "${_filepath}"
    if [ -f "${_filename}" ]
    then
        f_PrintLog "Warn" "file ${_filename} already exists, rename it now!!"
        ls -l ${_filename}
        sleep 1
        mv "${_filename}" "${_filepath}/${_filename##*/}.bak_$(date +%Y%m%d.%H%M%S)"
    fi
}

function f_check_runtime
{
    _so_config=$1
    f_PrintLog "Info" "Start to run Script f_check_runtime"
    _ts1=$(awk -F= '{if($1=="PRIMARYRUNTIME"){gsub("\\.","",$2);printf "%d",$2;exit}}' ${_so_config} 2>/dev/null)
    _ts2=$(awk -F= '{if($1~"STANDBYRUNTIME"){gsub("\\.","",$2);printf "%d",$2;exit}}' ${_so_config} 2>/dev/null)

    f_PrintLog "Info" "Get runtime from config PRIMARYRUNTIME:${_ts1}, STANDBYRUNTIME:${_ts2}."
    
    curdt=$(date +%Y%m%d)
    if [ "${curdt}" != "${_ts1:0:8}" -o "${curdt}" != "${_ts2:0:8}" ]
    then
        f_PrintLog "Info" "The runtime ${_ts1:0:8},${_ts2:0:8} is not today ${curdt}, check failed!!!"
        exit 1
    fi

    # _tsdiff=$(expr ${_ts1} - ${_ts2})
    # if [ ${_tsdiff} -le 100 -a ${_tsdiff} -ge -100 ]
    # then
    #     f_PrintLog "Info" "The runtime diff is ${_tsdiff}, check pass!"
    # else
    #     f_PrintLog "Info" "The runtime diff is ${_tsdiff}, config file may wrong, check failed!!!"
    #     exit 1
    # fi

}

function f_checkpointnull
{
    _ip=$1
    f_PrintLog "Info" "Start to run Script f_checkpointnull"
    #检查是否存在checkpoint表
    ck_sql="select count(1) from information_schema.tables where table_name='checkpoint' and table_schema='${checkpointschema}'"
    f_PrintLog "Info" "START TO RUN SQL:${ck_sql}, DBinfo ${_ip}"
    exec_result=$(mysql -h${_ip} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${ck_sql}")
    if [ $exec_result -eq 0 ]
    then 
        f_PrintLog "Info" "Remote cluster ${clsName} checkpoint table not exists, check pass!"  
        return 0
    else
        f_PrintLog "Info" "Remote cluster ${clsName} checkpoint exists, continue check."  
    fi

    #checkpoint没有数据
    ck_sql="select count(1) from ${checkpointschema}.checkpoint"
    f_PrintLog "Info" "START TO RUN SQL:${ck_sql}, DBinfo ${_ip}"
    exec_result=$(mysql -h${_ip} -P${tidbPort} -u${user} -p${tidbPasswd:-123} -NBe "${ck_sql}")
    if [ $exec_result -gt 1 ]
    then 
        f_PrintLog "Error" "Remote cluster ${clsName} checkpoint contain rows, pls delete it, check failed!!!"  
        exit 1
    fi
}


###############################################################################
##Init##
export LANG=en_US
TidbHome=$(awk -F: -v un="tidb" '{if($1==un){print $6;exit}}' /etc/passwd 2>/dev/null)
NumProc=$(awk '/^processor[ |\t]*:/{c++}END{print c}' /proc/cpuinfo 2>/dev/null)
ShellName="$(echo $0|awk -F / '{print $NF}')";ShellOption="$@";
WorkDir="$(echo $0|sed s/${ShellName}//g)";[ -z "${WorkDir}" ] && WorkDir=${PWD};cd ${WorkDir};WorkDir=${PWD}
LogDir="${TidbHome}/switchover/logs";mkdir -p ${LogDir};
RunTs=$(date "+%Y%m%d.%H%M%S")
LogFile="${LogDir}/${script_base_name}_${ShellName}.${RunTs}.log";
Shellcksum=$(cksum ${WorkDir%/}/${ShellName} 2>/dev/null|awk '{printf "%d",$1}')
declare -i UserID=$(id -u);declare UserName=$(whoami);declare HostName=$(hostname);
###############################################################################
##TiDB Var##
locallogin="tidblogin"
adminusers="root,tidb,timon,repl"
checkpointschema="tidb_binlog"
backupdir="/data2/tidbbackup"
user="timon";pAsswd=""
tidbsoconfig="${TidbHome}/switchover/cluster_switchover.config"
[ -f ${tidbsoconfig} ] && source ${tidbsoconfig}
###############################################################################
###############################################################################
##Get inopt
[ -z "${ShellOption}" -a -z "${primaryhostname}" -a -z "${standbyhostname}" ] && f_syntax;
# [ -n "${ShellOption}" ] && f_syntax;
# [ -z "${ShellOption}" -a -z "${primaryhostname}" -a -z "${standbyhostname}" ] && f_syntax;
for inopt in ${ShellOption}
do
    case $(echo $inopt|tr a-z A-Z) in
    -PRIMARYHOSTNAME|-HOSTNAME) CurOpt="-HOSTNAME_P";continue;;
    -STANDBYHOSTNAME) CurOpt="-HOSTNAME_S";continue;;
    -HELP|--HELP|-H) f_syntax;;
    -*) CurOpt="";continue;;
    esac
    case "${CurOpt}" in
    -HOSTNAME_P) declare primaryhostname="${inopt}";continue;;
    -HOSTNAME_S) declare standbyhostname="${inopt}";continue;;
    esac
done

f_PrintLog "Info" "Start to run Script ${ShellName}, the Shellcksum is ${Shellcksum}"
f_PrintLog "Info" "Script Options:${ShellOption}"


function f_readandcheckconfig
{

    f_PrintLog "Info" "Start to run f_readandcheckconfig."
    #
    [ -f ${tidbsoconfig} ] && source ${tidbsoconfig} 
    appusers="${PRIMARYAPPUSERS}"
    tidbPasswd=$(echo "${PASSWD}="|base64 -d)
    tidbIp="${PRIMARYIP}"
    tidbIp_s="${STANDBYIP}"
    tidbPort="${PRIMARYPORT}"
    clsName="${PRIMARYCLUSTERNAME}"
    clsName_s="${STANDBYCLUSTERNAME}"
    tidbDrainerId="${PRIMARYT2TDRAINER%,*}"
    backupdir="${BACKUPDIR:-/tidbbackup}"
    
    f_PrintLog "Info" "Get ${HostRole} cluster info finished. clsname:${clsName}, clsname_s:${clsName_s}, Ip:${tidbIp}, standbyIp:${tidbIp_s}, drainerId:${tidbDrainerId}"
    
    f_PrintLog "Info" "Check config hostname and cluster name."
    clsNamelocal=$(tiup cluster list 2>/dev/null|awk '{if($3~"v" && $4~"tiup"){printf "%s\n",$1;exit}}') 
    if [ -z "${clsNamelocal}" ]
    then
        f_PrintLog "Error" "Can not find valid cluster name using:tiup cluster list"
        exit 1
    else
        f_PrintLog "Info" "Find valid local cluster: ${clsNamelocal}"
        tiup cluster display ${clsNamelocal} 
    fi

    if [ "${HostRole}" = "PRIMARY" -a "${clsName}" != "${clsNamelocal}" ]
    then
        f_PrintLog "Error" "Check config clsname failed, PRIMARY CLUSTERNAME: ${clsName} != ${clsNamelocal}, exit!!!"
        exit 1
    elif [ "${HostRole}" = "PRIMARY" -a "${PRIMARYHOSTNAME}" != "${HostName}" ]
    then
        f_PrintLog "Error" "Check config hostname failed, PRIMARY HOSTNAME: ${PRIMARYHOSTNAME} != ${HostName}, exit!!!"
        exit 1
    elif [ "${HostRole}" = "STANDBY" -a "${clsName_s}" != "${clsNamelocal}" ]
    then
        f_PrintLog "Error" "Check config clsname failed, STANDBY CLUSTERNAME: ${clsName_s} != ${clsNamelocal}, exit!!!"
        exit 1
    elif [ "${HostRole}" = "STANDBY" -a "${STANDBYHOSTNAME}" != "${HostName}" ]
    then
        f_PrintLog "Error" "Check config hostname failed, STANDBY HOSTNAME: ${STANDBYHOSTNAME} != ${HostName}, exit!!!"
        exit 1
    fi

    f_check_runtime "${tidbsoconfig}" 
}


function clean_checkpoint
{
    _ip=$1
    
    f_PrintLog "Info" "Start to run clean_checkpoint."
    #主备区分执行
    if [ "${_ip}" = "${tidbIpLocal}" ]
    then 
        connstr="--login-path=${locallogin:-tidblogin}"
    else
        connstr="-h${_ip} -P${tidbPort} -u${user} -p${tidbPasswd:-123}"
    fi

    ck_sql="select count(1) from information_schema.tables where table_name='checkpoint' and table_schema='${checkpointschema}'"
    f_PrintLog "Info" "START TO RUN SQL:${ck_sql}, DBinfo ${_ip}"
    exec_result=$(mysql ${connstr} -NBe "${ck_sql}")
    if [ $? -eq 0 -a "${exec_result:-1}" = "0" ]
    then
        f_PrintLog "Warn" "checkpoint table not exists, skip drop table ${checkpointschema}.checkpoint!"
        return 0
    fi

    f_committs_stop "${_ip}"
    f_drop_checkpoint "${_ip}"
    f_PrintLog "Info" "Drop table ${checkpointschema}.checkpoint finish."
}


function f_startrepl_main
{
    f_PrintLog "Info" "Start to run f_startrepl_main."
    
    #检查远程checkpoint表
    f_checkpointnull "${tidbIp}"

    #部署反向drainer
    tidbDrainerId=$(tiup cluster display ${clsName_s} 2>/dev/null|awk -v id="${tidbIp}" '{
        if($1~id && $2=="drainer"){printf "%s",$1;exit}
        }')
    
    if [ -z "${tidbDrainerId}" ]
    then
        f_PrintLog "Info" "Can not find valid drainer node, the cluster name is ${clsName_s}, start to scale-out."
        if [ -f "${TidbHome}/switchover/${clsName_s}_scaleout_slave_drainer.yaml" ]
        then
            f_PrintLog "Info" "Start to scale-out drainer, the config file is:${TidbHome}/${clsName_s}_scaleout_slave_drainer.yaml"
            f_sacleoutdrainer "${clsName_s}" "${TidbHome}/switchover/${clsName_s}_scaleout_slave_drainer.yaml"
        else
            f_PrintLog "Error" "Tiup scale-out yaml not exist, pls provide ${TidbHome}/switchover/${clsName_s}_scaleout_slave_drainer.yaml"
            exit 1
        fi
    else
        #启动反向drainer
        f_PrintLog "Info" "Find a remote drainer node, the cluster name is ${clsName_s}, start to start ${tidbDrainerId}"
        f_startdrainer "${clsName_s}" "${tidbDrainerId}" 
    fi

    #进行写入数据复制测试
    f_check_insert "${tidbIp_s}" "${tidbIp}"

    #应用用户授权
    f_grant_user "${tidbIp_s}" "${app_bakfile}"
}


function f_primary_waitstart_main
{
    [ "${HostRole}" = "STANDBY" ] && return 0
    f_PrintLog "Info" "Start to run f_primary_waitstart_main."
    startdrok="${TidbHome}/switchover/tmp_startdrok_standby"
    
    i=0
    while [ $i -le 30 ]
    do
        if [ -f "${startdrok}" ]
        then
            startrok_ts=$(grep "${STANDBYCLUSTERNAME} $(date +%Y%m%d)" ${startdrok} | wc -l)
            if [ ${stopdrok_ts:-0} -ge 1 ]
            then
                f_PrintLog "Info" "get startdrok file from standby host:${STANDBYHOSTNAME},${tidbIp_s}!"
                echo "************************"
                cat "${startdrok}"
                echo "************************"
                break
            else
                f_PrintLog "Info" "find incorrect startdrok file, continue wait..."                
                cat "${startdrok}"
            fi
        else
            f_PrintLog "Info" "wait startdrok file from standby host:${STANDBYHOSTNAME},${tidbIp_s}, sleep 5, i=${i}"            
        fi
        i=$((i+1))
        sleep 5
    done
    
    if [ ! -f "${startdrok}" -o ${startrok_ts:-0} -eq 0 ]
    then
        f_PrintLog "Error" "Can not get startdrok file from standby host:${STANDBYHOSTNAME},${tidbIp_s}, not grant app user privileges, exit!!!"
        exit 1
    fi

    # revoke repl user standby
    f_runsqlfile "${app_rvkfile}" "${tidbIp}" 
    _rt_code=$?
    if [ ${_rt_code} -eq 0 ]
    then
        f_PrintLog "Info" "grant app user privileges finish!"
    else
        f_PrintLog "Error" "grant app user privileges failed!!!"
        exit 1
    fi

    f_PrintLog "Info" "rm stopdrok file:${startdrok} now!"
    echo "************************"
    cat ${startdrok}
    echo "************************"
    rm -f ${startdrok}

}


function f_rmconfig_main
{
    f_PrintLog "Info" "Start to run f_rmconfig_main."
    f_PrintLog "Info" "Start to run rm config file ${tidbsoconfig}"
    f_backfile "${tidbsoconfig}"
    f_backfile "${TidbHome}/switchover/cluster_switchover_config_P.cnf"
    f_backfile "${TidbHome}/switchover/cluster_switchover_config_S.cnf"
}


###################
####    Main   ####
###################


function f_standby_startrepl_main
{
    f_PrintLog "Info" "Start to run Script ${ShellName} on ${HostName}, script_base_name ${script_base_name}."

    if [ -f ${tidbsoconfig} ] 
    then
        f_PrintLog "Info" "Start to buildin config variables, ${tidbsoconfig}."
        source ${tidbsoconfig}
        curdt=$(date +%Y%m%d)
        if [ -z $(echo ${PRIMARYRUNTIME} | grep ${curdt}) ]
        then
            f_PrintLog "Error" "Invalid config file, RUNTIME:${PRIMARYRUNTIME}, ${tidbsoconfig}."
            exit 1
        fi
    else
        f_PrintLog "Error" "Can not find valid config, ${tidbsoconfig}, exit"
        exit 1
    fi 

    #local var
    localProc=$(ps -ef | grep "tidb-server -P 4000"| grep -v grep)
    localPort=$(echo ${localProc}|awk -F"-P" '{print $2}'|awk '{printf "%d",$1}')
    localIp=$(echo ${localProc}|awk -F"--advertise-address=" '{print $2}'|awk '{printf "%s",$1}')
    tidbIpLocal="${localIp}"

    #check script var
    #check host role
    if [ "${primaryhostname}" != "${HostName}" -a "${standbyhostname}" != "${HostName}" ] 
    then
        f_PrintLog "Error" "InputHost are ${primaryhostname} and ${standbyhostname}, but now i am ${HostName}, exit"
        exit 1
    fi
    
    if [ "${primaryhostname}" = "${HostName}" ] 
    then
        HostRole="PRIMARY"
    elif [ "${standbyhostname}" = "${HostName}" ]
    then
        HostRole="STANDBY"
    else
        HostRole="UNKNOWN"
    fi
    
    if [ "${PRIMARYHOSTNAME}" != "${HostName}" -a "${STANDBYHOSTNAME}" != "${HostName}" ]
    then
        f_PrintLog "Info" "Current hostname is ${HostName}, the script need to run on  ${PRIMARYHOSTNAME}, quit"
        exit 0
    fi

    #获取脚本执行需要的集群信息
    f_readandcheckconfig  
    f_PrintLog "Info" "Get ${HostRole} cluster info finished. clsname:${clsName}, clsname_s:${clsName_s}, Ip:${tidbIp}, standbyIp:${tidbIp_s}, drainerId:${tidbDrainerId}"
    
    #grant sql files
    grantsqldir="${TidbHome}/switchover/grantsql"
    # mkdir -p ${grantsqldir}
    app_bakfile=${grantsqldir}"/usergrant_app_${tidbIp}_$(date +%Y%m%d).sql"
    app_rvkfile=${grantsqldir}"/userrevoke_app_${tidbIp}_$(date +%Y%m%d).sql"
    repl_bakfile=${grantsqldir}"/usergrant_repl_${tidbIp_s}_$(date +%Y%m%d).sql"
    repl_rvkfile=${grantsqldir}"/userrevoke_repl_${tidbIp_s}_$(date +%Y%m%d).sql"

    if [ "${HostRole}" = "PRIMARY" ]
    then
        #主集群执行repl用户回收权限
        f_grant_user "${tidbIp}" "${repl_bakfile}"
        rt_code=$?
        
        f_rmconfig_main

        #主集群等待本地drainer进程启动后,执行app权限回收,最多等待5分钟
        # f_primary_waitstart_main
        f_PrintLog "Info" "Script ${script_base_name} run on ${HostRole} Node finished, quit."
        exit $rt_code
    fi

    #删除备库上的checkpoint表
    [ -z "${checkpointschema}" ] && checkpointschema="tidb_binlog"
    clean_checkpoint "${tidbIp_s}"  

    #检查备库的binlog状态
    f_checkbinlog "${clsName_s}" 

    #部署反向drainer
    f_startrepl_main

    #删除配置文件
    f_rmconfig_main
    f_PrintLog "Info" "Successful to run Script ${ShellName} on ${HostName}, script_base_name ${script_base_name}."

}
 
f_standby_startrepl_main | tee -a ${LogFile}
exit ${PIPESTATUS[0]}