#/!bin/bash
source ~/.bash_profile

#========== parameter ==============
CLUSTER_NAME=nonghang2
# unhealthy store_id list
STORE_ID="2,7"
#tools dir for tikv-ctl
TIKV_CTL_DIR=~/.tiup/components/ctl/v4.0.10
#tikv-servers information 
TIKV_IP_DATA_LIST=`tiup cluster display ${CLUSTER_NAME} -R tikv|grep tikv |grep -v tiup|awk '{print $3","$7}'|xargs`
#pd connection information
PD_CONN=`tiup cluster display ${CLUSTER_NAME} |grep pd|grep -v tiup|awk '{print $1}'`
#pd deploy dir
PD_DEPLOY_DIR=`tiup cluster display ${CLUSTER_NAME} |grep pd|grep -v tiup|awk '{print $8}'`


if [ 'x'${STORE_ID} == 'x' ]
then 
	echo " usage : sh unsafe_recover.sh  primary_store_id[,primary_store_id][,.....]"
	exit 1
fi


#stop cluster 
tiup cluster stop ${CLUSTER_NAME} 

# 1 remove unhealthy store id from tidb cluster 
echo "=====================================================start  unsafe recover======================================================================="
echo " ------------------------------------------------- 1. remove all unhealty tikv store id from cluster ------------------------------------------------"
for i in $TIKV_IP_DATA_LIST
do
echo "copy tikv-ctl to "${i%,*}":"
scp ${TIKV_CTL_DIR}/tikv-ctl ${i%,*}:~/

echo "remove unhealthy  store id :"
ssh ${i%,*} ./tikv-ctl --db ${i#*,}/db unsafe-recover remove-fail-stores -s ${STORE_ID} --all-regions
echo "remove finished"
done 

# 2 change pd force new cluster 
echo " ------------------------------------------------- 2. force new cluster mode start pd server ------------------------------------------------"
tiup cluster exec ${CLUSTER_NAME} -R pd --command "sed -i '/pd-server/ s#pd-server#pd-server --force-new-cluster#' ${PD_DEPLOY_DIR}/scripts/run_pd.sh"


#start pd 
tiup cluster start ${CLUSTER_NAME} -N ${PD_CONN}


echo " ------------------------------------------------- 3. force set max-replicas to 1   -----------------------------------------------"
#change replicas to 1
tiup ctl pd -u http://${PD_CONN} config set max-replicas 1


echo " ------------------------------------------------- 4. reload cluster    -----------------------------------------------"
# relaod cluster 
tiup cluster stop  ${CLUSTER_NAME}
tiup cluster start ${CLUSTER_NAME}



echo " ------------------------------------------------- 5. scale out monitored servers -----------------------------------------------"
#scale out
cat > scale-out-monitor.yaml <<EOF  
monitoring_servers:
- host: ${PD_CONN%:*}
grafana_servers:
- host: ${PD_CONN%:*}
EOF


#scale out monitor
tiup cluster scale-out ${CLUSTER_NAME} scale-out-monitor.yaml  -y

echo "=====================================================unsafe recover finished ======================================================================="