#!/usr/bin/env bash set -xeou pipefail mode=${1:-"classic"} # create shared bucket function create_shared_bucket() { set +e # error to continue local region=$1 local bucketName=$2 local env=$3 local account=$4 # 检查存储桶是否存在 head_bucket_command=$(aws s3api head-bucket --region "${region}" --bucket "${bucketName}" --profile "${account}" 2>&1) if echo "$head_bucket_command" | grep -q "HeadBucket operation: Not Found"; then # 存储桶不存在，创建存储桶 aws s3api create-bucket \ --bucket "${bucketName}" \ --create-bucket-configuration LocationConstraint="${region}" \ --acl private \ --region "${region}" \ --profile "${account}" \ --no-cli-pager if test $? -ne 0; then echo "failed to create bucket ${bucketName}" return 1 fi aws s3api put-bucket-tagging \ --bucket "${bucketName}" \ --profile "${account}" \ --tagging "TagSet=[ {Key=Name,Value=${bucketName}}, {Key=environment,Value=${env}}, {Key=servicetype,Value=dedicated}, {Key=usedby,Value=customer-tidb}, {Key=creator,Value=auto-deploy}, ]" if test $? -ne 0; then echo "failed to set tags for bucket ${bucketName}" return 1 fi elif echo "$head_bucket_command" | grep -q "HeadBucket operation: Forbidden"; then # 存储桶已被其他账户创建 echo "bucket ${bucketName} has created by other account" return 1 else # 存储桶已存在 echo "bucket ${bucketName} has already created" fi return 0 } ROOT=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) source "$ROOT/lib.sh" remote_config=${CONFIG_PATH} local_config=/tmp/base-config.yaml bucket_name=${remote_config#s3://} bucket_name=${bucket_name%%/*} output_bucket_region=$(aws s3api get-bucket-location --bucket=$bucket_name --query="LocationConstraint" \ --region=us-west-2 --output=text --no-cli-pager) aws s3 cp "${remote_config}" "${local_config}" --region=$output_bucket_region argocd_repo=tidbcloud/app-delivery region=$(yq '.region' "${local_config}") logicalRegionName=$(yq '.name' "${local_config}") env=$(yq '.env' "${local_config}") cloud=$(yq '.cloudProvider' "${local_config}") cluster=$(yq '.k8s.name' "${local_config}") jobId=$(yq '.jobId' "${local_config}") output=$(yq '.outputPath' "${local_config}") imageRegistry=$(yq '.imageRegistry' "${local_config}") regionalControlplaneIPs=$(yq -ojson '.k8s.publicIps | map(. + "/32")' "$local_config") # eg: ["52.79.68.176/32", "3.35.39.51/32"] availabilityZones=$(yq -ojson '.jobOptions.regionalServer.availabilityZones | split(",")' "${local_config}") # eg: ["us-west-2a", "us-west-2b", "us-west-2c"] defaultAwsAccount=$(yq '.jobOptions.regionalServer.defaultDataplaneAccount' "${local_config}") # eg: 123456789012 dataplaneAccounts=$(yq -ojson '.aws.dataplaneAccounts' "${local_config}") # eg: ["123456789012", "234567890123"] auth0Audience=$(yq '.jobOptions.regionalServer.auth0Audience' "${local_config}") # eg: prod-proxy-service auth0Domain=$(yq '.jobOptions.regionalServer.auth0Domain' "${local_config}") # eg: tidb.auth0.com globalControlplaneIPs=$(yq -ojson '.jobOptions.regionalServer.globalControlplaneIPs' "${local_config}") # eg: ["44.231.162.131/32", "100.20.198.224/32"] o11yRegionalServiceIPs=$(yq -ojson '.jobOptions.regionalServer.o11yRegionalServiceIPs' "${local_config}") # eg: ["1.2.3.5/32", "3.4.5.6/32"] clinicIPs=$(yq -ojson '.jobOptions.regionalServer.clinicIPs' "${local_config}") # eg: ["3.5.7.7/32"] bastionIPs=$(yq -ojson '.jobOptions.regionalServer.bastionIPs' "${local_config}") # eg: ["6.4.6.4/32"] publicHostedZoneDomain=$(yq '.jobOptions.regionalServer.publicHostedZoneDomain' "${local_config}") # eg: tidbcloud.com publicHostedZoneID=$(yq '.jobOptions.regionalServer.publicHostedZoneID' "${local_config}") # eg: Z1234567890 o11yRegionalServiceEndpoint=$(yq '.jobOptions.regionalServer.o11yRegionalServiceEndpoint' "${local_config}") # eg: https://o11y.tidbcloud.com pcaArn=$(yq '.jobOptions.clusterServer.fullManage.pcaArn' "${local_config}") sharedBucketIndexes=$(yq -ojson '.jobOptions.regionalServer.sharedBucketIndexes // ["000"]' "${local_config}") # eg: ["000"] remote_iam_config=$(yq '.jobOutput.iam' "${local_config}") local_iam_config=/tmp/iam-config.yaml aws s3 cp ${remote_iam_config} ${local_iam_config} --region=$output_bucket_region awsIrsaRoleArn=$(yq '.regionServer.roleArn' "${local_iam_config}") defaultAwsDp4DrsRole=$(yq '.regionServer.dataplaneRoleName' "${local_iam_config}") awsAssumeRoleGlobalArn=$(yq '.regionServer.globalRoleArn' "${local_iam_config}") eksServiceRoleName=$(yq '.regionServer.eksServiceRoleName' "${local_iam_config}") clusterControllerAssumeRoleName=$(yq '.clusterController.dataplaneRoleName' "${local_iam_config}") readonlyClusterControllerAssumeRoleName=$(yq '.clusterController.readonlyDataplaneRoleName' "${local_iam_config}") dpControllerAssumeRoleName=$(yq '.dpProvider.dataplaneRoleName' "${local_iam_config}") ticdcControllerAssumeRoleName=$(yq '.ticdcService.dataplaneRoleName' "${local_iam_config}") o11y_enabled=$(yq '.jobOptions.o11y.enabled' "${local_config}") o11y_cluster_id="" if [ "${o11y_enabled}" == "true" ]; then remote_o11y_agent_output=$(yq '.jobOutput.o11y-agent' "${local_config}") local_o11y_agent_output=/tmp/o11y-agent-output.yaml aws s3 cp ${remote_o11y_agent_output} ${local_o11y_agent_output} --region=$output_bucket_region o11y_cluster_id=$(yq '.o11y_cluster_id' "${local_o11y_agent_output}") fi fullManagerServiceRoles=() count=$(echo "$dataplaneAccounts" | jq "length") for (( i=0; i < $count ; i++ )); do acct=$(echo "$dataplaneAccounts" | jq -r ".[$i]") fullManagerServiceRoles+=( "arn:aws:iam::$acct:role/$dpControllerAssumeRoleName" ) done fullManagerServiceRolesJson=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${fullManagerServiceRoles[@]}") # authorizedSubjects is a list of subject that is allowed to access the regional server # eg: [ # "arn:aws:iam::557537366020:role/regional-server-role-o11y-prod-regional-ap-northeast-2-36e136b", # o11y role # "arn:aws:iam::385595570414:role/dev-us-west-2-ng-cluster-ng-service", # ng cluster service role # "arn:aws:iam::380838443567:role/prod-ap-northeast-2-cluster-controller", # cluster controller role # "arn:aws:iam::380838443567:role/prod-ap-northeast-2-dm-controller", # dm controller role # "arn:aws:iam::380838443567:role/prod-ap-northeast-2-import-controller", # import controller role # "arn:aws:iam::730335266318:role/full-manager-service-role", # manager service role # "arn:aws:iam::380838443567:role/full-manager-service-role", # manager service role # "arn:aws:iam::886436925895:role/dev-us-west-2-ng-cluster-ng-service-assumed-role", # role for ng cluster server to assume dataplane # "arn:aws:iam::730335266318:role/prod-ap-northeast-2-cluster-controller-assumed-role", # role for cluster controller to assume dataplane # "arn:aws:iam::730335266318:role/prod-ap-northeast-2-dm-controller-assumed-role", # role for dm controller to assume dataplane # "arn:aws:iam::730335266318:role/prod-ap-northeast-2-import-controller-assumed-role", # role for import controller to assume dataplane # "arn:aws:iam::380838443567:role/devops-server-prod", # devops server role # ] clusterControllerRole=$(yq ".clusterController.roleArn" "${local_iam_config}") ticdcServiceControllerRole=$(yq ".ticdcService.controllerRoleArn" "${local_iam_config}") ticdcServiceServerRole=$(yq ".ticdcService.serverRoleArn" "${local_iam_config}") clusterDataplaneRoles=$(yq -ojson ".clusterController.dataplaneRoleArns" "${local_iam_config}") dmControllerRole=$(yq ".tcidm.dmRoleArn" "${local_iam_config}") importControllerRole=$(yq ".tcidm.importRoleArn" "${local_iam_config}") dmDataplaneRoles=$(yq ".tcidm.dmDataplaneRoleArns" "${local_iam_config}") importDataplaneRoles=$(yq ".tcidm.importDataplaneRoleArns" "${local_iam_config}") authorizedSubjects=$(yq -ojson -e ".jobOptions.regionalServer.authorizedSubjects + \ [\"$clusterControllerRole\", \"$ticdcServiceControllerRole\", \"$ticdcServiceServerRole\", \"$dmControllerRole\", \"$importControllerRole\"] + \ $fullManagerServiceRolesJson + \ $clusterDataplaneRoles + \ $dmDataplaneRoles + \ $importDataplaneRoles" "${local_config}") # currently we only have one shared bucket for each account in each region declare -A acct2shared_buckets for idx in $(echo "$sharedBucketIndexes" | jq -r '.[]'); do for acct in $(echo "$dataplaneAccounts" | jq -r '.[]'); do shared_bucket_name="${env}-sb-${acct}-${logicalRegionName}-${idx}" create_shared_bucket "${region}" "${shared_bucket_name}" "${env}" "${acct}" if [[ ! -v acct2shared_buckets[$acct] ]]; then acct2shared_buckets[$acct]="$shared_bucket_name" else acct2shared_buckets[$acct]="${acct2shared_buckets[$acct]} $shared_bucket_name" fi done done acct2shared_buckets_json_str="{" for key in "${!acct2shared_buckets[@]}"; do acct2shared_buckets_json_str+="\"$key\":\"${acct2shared_buckets[$key]}\"," done acct2shared_buckets_json_str="${acct2shared_buckets_json_str%,}" acct2shared_buckets_json_str+="}" is_next_gn=false byoc="null" if [[ $mode == "byoc-ng" ]]; then is_next_gn=true vpc_id=$(aws eks describe-cluster \ --name $cluster \ --query "cluster.resourcesVpcConfig.vpcId" \ --output text) echo "found vpc: $vpc_id" cluster_sg=$(aws eks describe-cluster \ --name $cluster \ --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" \ --output text) private_subnets=$(aws ec2 describe-subnets \ --filters "Name=vpc-id,Values=$vpc_id" \ --query "Subnets[?Tags[?Key=='kubernetes.io/role/internal-elb']].SubnetId" \ --output json) byoc=$(cat <