#!/usr/bin/env python3 import json import subprocess def seed_ctx(region): return f"controlplane.seed-aws-{region}" def o11y_ctx(region): return f"controlplane.o11y-prod-regional-{region}" def run(command: str): print("# [Running]:", command) return run_raw(command.split()) def run_raw(command): try: return subprocess.run(command, capture_output=True, text=True, check=True) except subprocess.CalledProcessError as e: print(f"Error: {e}") print(f"stdout: {e.stdout}") print(f"stderr: {e.stderr}") raise def get_seed_vpc(region, project_id): result = run( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} get vpc -l tags.tidbcloud.com/project={project_id} -o json" ) appboxes = json.loads(result.stdout) if len(appboxes["items"]) > 1: raise ValueError("Multiple vpcs found") elif len(appboxes["items"]) == 0: raise ValueError("No vpcs found") return appboxes["items"][0] def get_seed_appbox(region, project_id): result = run( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} get appboxes -l tags.tidbcloud.com/project={project_id} -o json" ) appboxes = json.loads(result.stdout) if len(appboxes["items"]) > 1: raise ValueError("Multiple appboxes found") elif len(appboxes["items"]) == 0: raise ValueError("No appboxes found") return appboxes["items"][0] def get_seed_admin_nodegroup(region, project_id, vpc_name): result = run( f"kubectl --context {seed_ctx(region)} get nodegroup -l tidbcloud-network={vpc_name} -o json" ) nodegroups = json.loads(result.stdout) for item in nodegroups["items"]: if item["metadata"]["name"].startswith("tidbcloud-dedicated-"): return item result = run( f"kubectl --context {seed_ctx(region)} get nodegroup -l tidbcloud-namespace=prod-{project_id} -o json" ) nodegroups = json.loads(result.stdout) for item in nodegroups["items"]: if item["metadata"]["name"].startswith(f"tidbcloud-prod-{project_id}"): return item raise ValueError("Admin nodegroup not found") def get_o11y_private_endpoint(region, tenant_id, project_id): result = run( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} get privateendpoint -l observability.pingcap.com/projectID={project_id} -o json" ) private_endpoints = json.loads(result.stdout) if len(private_endpoints["items"]) > 1: raise ValueError("Multiple private endpoints found") elif len(private_endpoints["items"]) == 0: raise ValueError("No private endpoints found") return private_endpoints["items"][0] def get_o11y_cluster(region, tenant_id, project_id, cluster_id): result = run( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} get cluster -l observability.pingcap.com/applicationID={cluster_id},observability.pingcap.com/projectID={project_id} -o json" ) clusters = json.loads(result.stdout) if len(clusters["items"]) == 0: print(f"No cluster found with {project_id}/{cluster_id}") return elif len(clusters["items"]) > 1: print(f"Multiple clusters found with {project_id}/{cluster_id}") return return clusters["items"][0] def get_o11y_o11yagent(region, tenant_id, project_id, cluster_id): result = run( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} get o11yagent -l tags.tidbcloud.com/cluster={cluster_id},tags.tidbcloud.com/project={project_id} -o json" ) agents = json.loads(result.stdout) if len(agents["items"]) == 0: print(f"No cluster found with {project_id}/{cluster_id}") return elif len(agents["items"]) > 1: print(f"Multiple clusters found with {project_id}/{cluster_id}") return return agents["items"][0] def get_dataplane_vmagent_pod(cluster_id): result = run( f"kubectl --context dedicated.{cluster_id} -n tidb{cluster_id} get pod -l app.kubernetes.io/name=victoria-metrics-agent -o json" ) pods = json.loads(result.stdout) if len(pods["items"]) == 0: print(f"No vmagent pod found with {cluster_id}") return elif len(pods["items"]) > 1: print(f"Multiple vmagent pod found with {cluster_id}") return return pods["items"][0] def gen_vpc_endpoint( region, project_id, vpc_id, subnet_ids, security_group_ids, service_name, provider_config_name, ): return f""" apiVersion: ec2.aws.crossplane.io/v1alpha1 kind: VPCEndpoint metadata: name: o11yd-migrated-{project_id} spec: deletionPolicy: Delete forProvider: region: {region} securityGroupIds: {json.dumps(security_group_ids)} serviceName: {service_name} subnetIds: {json.dumps(subnet_ids)} privateDNSEnabled: false tagSpecifications: - resourceType: vpc-endpoint tags: - key: Vendor value: PingCAP - key: Name value: o11yd-migrated-{project_id} vpcEndpointType: Interface vpcId: {vpc_id} managementPolicies: - "*" providerConfigRef: name: "{provider_config_name}" """ def gen_k8s_nodegroup( region, project_id, k8s_cluster_name, node_role, subnet_ids, version, release_version, provider_config_name, ami_type="BOTTLEROCKET_x86_64", ): return f""" apiVersion: eks.aws.crossplane.io/v1alpha1 kind: NodeGroup metadata: labels: o11y.tidbcloud.com/component: o11y o11y.tidbcloud.com/project: "{project_id}" name: tidbcloud-o11y-k8s-{project_id} spec: deletionPolicy: Delete forProvider: amiType: {ami_type} capacityType: ON_DEMAND clusterName: "{k8s_cluster_name}" diskSize: 20 instanceTypes: - t3.medium labels: o11y.tidbcloud.com/component: o11y o11y.tidbcloud.com/project: "{project_id}" nodeRole: "{node_role}" region: "{region}" releaseVersion: "{release_version}" scalingConfig: desiredSize: 1 maxSize: 3 minSize: 1 subnets: {json.dumps(subnet_ids)} tags: Name: tidbcloud-o11y-k8s-{project_id} tidbcloud.com/nodegroup-name: tidbcloud-o11y-k8s-{project_id} taints: - effect: NO_EXECUTE key: o11y-dedicated value: o11y - effect: NO_EXECUTE key: o11y-project value: "{project_id}" updateConfig: force: false maxUnavailable: 1 version: "{version}" managementPolicies: - "*" providerConfigRef: name: "{provider_config_name}" """ def gen_tidb_nodegroup( region, cluster_id, k8s_cluster_name, node_role, subnet_ids, version, release_version, provider_config_name, ami_type="BOTTLEROCKET_x86_64", ): return f""" apiVersion: eks.aws.crossplane.io/v1alpha1 kind: NodeGroup metadata: labels: o11y.tidbcloud.com/cluster: "{cluster_id}" o11y.tidbcloud.com/component: o11y name: tidbcloud-o11y-tidb-{cluster_id} spec: deletionPolicy: Delete forProvider: amiType: {ami_type} capacityType: ON_DEMAND clusterName: "{k8s_cluster_name}" diskSize: 20 instanceTypes: - t3.medium labels: o11y.tidbcloud.com/cluster: "{cluster_id}" o11y.tidbcloud.com/component: o11y nodeRole: "{node_role}" region: "{region}" releaseVersion: "{release_version}" scalingConfig: desiredSize: 1 maxSize: 3 minSize: 1 subnets: {json.dumps(subnet_ids)} tags: Name: tidbcloud-o11y-tidb-{cluster_id} tidbcloud.com/nodegroup-name: tidbcloud-o11y-tidb-{cluster_id} taints: - effect: NO_EXECUTE key: o11y-dedicated value: o11y - effect: NO_EXECUTE key: o11y-cluster value: "{cluster_id}" updateConfig: force: false maxUnavailable: 1 version: "{version}" managementPolicies: - "*" providerConfigRef: name: "{provider_config_name}" """ def gen_o11y_k8s_agent( region, tenant_id, project_id, k8s_cluster_name, account_id, vpc_id, security_group_id, node_role, node_subnet_id, node_subnet_ids, node_vpc_name, node_zone, node_version, node_release_version, provider_config_name, node_ami_type="BOTTLEROCKET_x86_64", ): return f""" apiVersion: agent.observability.pingcap.com/v2 kind: O11yK8sAgent metadata: annotations: # o11y.cloud.pingcap.com/id: o11y_id # o11y.cloud.pingcap.com/vm-account-id: "vm_account_id" # o11y.cloud.pingcap.com/vm-project-id: "vm_project_id" # o11y.cloud.pingcap.com/agent-role-arn: agent_role # o11y.cloud.pingcap.com/log-bucket-name: bucket # o11y.cloud.pingcap.com/endpoint-service-name: endpoint_service_name # o11y.cloud.pingcap.com/service-dns-name: promethues-enpoint-a.observability.tidbcloud.com # o11y.cloud.pingcap.com/service-origin-dns-name: promethues-enpoint-a.observability.tidbcloud.com # o11y.cloud.pingcap.com/status: ACTIVE o11y.cloud.pingcap.com/version: "0" o11y.cloud.pingcap.com/disable-private-endpoint-private-dns: "true" o11y.cloud.pingcap.com/o11y-service-interface-endpoint-name: o11yd-migrated-{project_id} o11y.cloud.pingcap.com/node-ami-type: {node_ami_type} o11y.cloud.pingcap.com/node-capacity-type: ON_DEMAND o11y.cloud.pingcap.com/node-desired-size: "1" o11y.cloud.pingcap.com/node-disk-size: "20" o11y.cloud.pingcap.com/node-instance-type: t3.medium o11y.cloud.pingcap.com/node-max-size: "3" o11y.cloud.pingcap.com/node-min-size: "1" o11y.cloud.pingcap.com/node-provider-config-reference: "{provider_config_name}" o11y.cloud.pingcap.com/node-release-version: "{node_release_version}" o11y.cloud.pingcap.com/node-role: "{node_role}" o11y.cloud.pingcap.com/node-subnet: "{node_subnet_id}" o11y.cloud.pingcap.com/node-version: "{node_version}" o11y.cloud.pingcap.com/node-vpc-name: "{node_vpc_name}" o11y.cloud.pingcap.com/node-zone: "{node_zone}" name: "{project_id}" namespace: observability spec: eks: accountId: "{account_id}" assumeExternalId: "123" assumeRoleArn: arn:aws:iam::{account_id}:role/prod-{region}-dp4drs nodeSubnetIDs: {json.dumps(node_subnet_ids)} providerConfigRef: name: "{provider_config_name}" securityGroupId: "{security_group_id}" vpcId: "{vpc_id}" kubernetesClusterName: "{k8s_cluster_name}" o11y: applicationId: k8s-infra bizType: tidbcloud projectId: "{project_id}" tenantId: "{tenant_id}" """ def gen_o11y_tidb_agent( region, tenant_id, project_id, cluster_id, k8s_cluster_name, account_id, vpc_id, security_group_id, node_role, node_subnet_id, node_subnet_ids, node_vpc_name, node_zone, node_version, node_release_version, provider_config_name, node_ami_type="BOTTLEROCKET_x86_64", ): return f""" apiVersion: agent.observability.pingcap.com/v2 kind: O11yTiDBAgent metadata: annotations: # o11y.cloud.pingcap.com/id: 574cf501-a8f1-4e39-93db-4035fb7753e5 # o11y.cloud.pingcap.com/vm-account-id: "6309" # o11y.cloud.pingcap.com/vm-project-id: "0" # o11y.cloud.pingcap.com/agent-role-arn: arn:aws:iam::557537366020:role/o11yA-e8cwDiaajKKATbZ69ZYy5f # o11y.cloud.pingcap.com/log-bucket-name: o11y-prod-shared-eu-central-1 # o11y.cloud.pingcap.com/endpoint-service-name: com.amazonaws.vpce.eu-central-1.vpce-svc-03ccc55aa7a8c95a4 # o11y.cloud.pingcap.com/service-dns-name: promethues-enpoint-a.observability.tidbcloud.com # o11y.cloud.pingcap.com/service-origin-dns-name: promethues-enpoint-a.observability.tidbcloud.com # o11y.cloud.pingcap.com/status: ACTIVE o11y.cloud.pingcap.com/version: "0" o11y.cloud.pingcap.com/disable-private-endpoint-private-dns: "true" o11y.cloud.pingcap.com/o11y-service-interface-endpoint-name: o11yd-migrated-{project_id} o11y.cloud.pingcap.com/node-ami-type: {node_ami_type} o11y.cloud.pingcap.com/node-capacity-type: ON_DEMAND o11y.cloud.pingcap.com/node-desired-size: "1" o11y.cloud.pingcap.com/node-disk-size: "20" o11y.cloud.pingcap.com/node-instance-type: t3.medium o11y.cloud.pingcap.com/node-max-size: "3" o11y.cloud.pingcap.com/node-min-size: "1" o11y.cloud.pingcap.com/node-provider-config-reference: "{provider_config_name}" o11y.cloud.pingcap.com/node-release-version: "{node_release_version}" o11y.cloud.pingcap.com/node-role: "{node_role}" o11y.cloud.pingcap.com/node-subnet: "{node_subnet_id}" o11y.cloud.pingcap.com/node-version: "{node_version}" o11y.cloud.pingcap.com/node-vpc-name: "{node_vpc_name}" o11y.cloud.pingcap.com/node-zone: "{node_zone}" name: "{cluster_id}" namespace: observability spec: clusterTopology: pd: cpu: "0" memory: "0" replicas: 3 tidb: cpu: "0" memory: "0" replicas: 1 tiflash: cpu: "0" memory: "0" replicas: 0 tikv: cpu: "0" memory: "0" replicas: 3 version: unknown eks: accountId: "{account_id}" assumeExternalId: "" assumeRoleArn: arn:aws:iam::{account_id}:role/prod-{region}-dp4drs nodeSubnetIDs: {json.dumps(node_subnet_ids)} providerConfigRef: name: "{provider_config_name}" securityGroupId: "{security_group_id}" vpcId: "{vpc_id}" kubernetesClusterName: "{k8s_cluster_name}" o11y: applicationId: "{cluster_id}" bizType: tidbcloud projectId: "{project_id}" tenantId: "{tenant_id}" """ def make_backup_files(region, tenant_id): run(f"mkdir -p backup/{region}/{tenant_id}") result = run( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} get o11yroute -l tags.tidbcloud.com/tenant={tenant_id} -o yaml" ) with open(f"backup/{region}/{tenant_id}/o11yroutes.yaml", "w") as f: f.write(result.stdout) result = run( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} get privateendpoint -o yaml" ) with open(f"backup/{region}/{tenant_id}/privateendpoints.yaml", "w") as f: f.write(result.stdout) result = run( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} get o11yagent -o yaml" ) with open(f"backup/{region}/{tenant_id}/o11yagents.yaml", "w") as f: f.write(result.stdout) result = run( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} get cluster -o yaml" ) with open(f"backup/{region}/{tenant_id}/clusters.yaml", "w") as f: f.write(result.stdout) def make_k8s_files(region, tenant_id, project_id): vpc = get_seed_vpc(region, project_id) admin_nodegroup = get_seed_admin_nodegroup( region, project_id, vpc["metadata"]["name"] ) private_endpoint = get_o11y_private_endpoint(region, tenant_id, project_id) vpc_endpoint_yaml = gen_vpc_endpoint( region=region, project_id=project_id, vpc_id=private_endpoint["spec"]["aws"]["vpcID"], subnet_ids=private_endpoint["spec"]["aws"]["subnetIDs"], security_group_ids=private_endpoint["spec"]["aws"]["securityGroupIDs"], service_name=private_endpoint["spec"]["aws"]["serviceName"], provider_config_name=admin_nodegroup["spec"]["providerConfigRef"]["name"], ) with open(f"k8s-vpc-endpoint-{project_id}.yaml", "w") as f: f.write(vpc_endpoint_yaml) k8s_nodegroup_yaml = gen_k8s_nodegroup( region=region, project_id=project_id, k8s_cluster_name=admin_nodegroup["spec"]["forProvider"]["clusterName"], node_role=admin_nodegroup["spec"]["forProvider"]["nodeRole"], subnet_ids=[vpc["status"]["nodeSubnets"][0]["subnetId"]], version=admin_nodegroup["spec"]["forProvider"]["version"], release_version=admin_nodegroup["spec"]["forProvider"]["releaseVersion"], provider_config_name=admin_nodegroup["spec"]["providerConfigRef"]["name"], ami_type=admin_nodegroup["spec"]["forProvider"]["amiType"], ) with open(f"k8s-nodegroup-{project_id}.yaml", "w") as f: f.write(k8s_nodegroup_yaml) o11y_k8s_agent_yaml = gen_o11y_k8s_agent( region=region, tenant_id=tenant_id, project_id=project_id, k8s_cluster_name=admin_nodegroup["spec"]["forProvider"]["clusterName"], account_id=private_endpoint["spec"]["aws"]["accountID"], vpc_id=private_endpoint["spec"]["aws"]["vpcID"], security_group_id=private_endpoint["spec"]["aws"]["securityGroupIDs"][0], node_role=admin_nodegroup["spec"]["forProvider"]["nodeRole"], node_subnet_id=vpc["status"]["nodeSubnets"][0]["subnetId"], node_subnet_ids=[item["subnetId"] for item in vpc["status"]["nodeSubnets"]], node_vpc_name=vpc["metadata"]["name"], node_zone=vpc["status"]["nodeSubnets"][0]["availabilityZone"], node_version=admin_nodegroup["spec"]["forProvider"]["version"], node_release_version=admin_nodegroup["spec"]["forProvider"]["releaseVersion"], provider_config_name=admin_nodegroup["spec"]["providerConfigRef"]["name"], node_ami_type=admin_nodegroup["spec"]["forProvider"]["amiType"], ) with open(f"k8s-agent-{project_id}.yaml", "w") as f: f.write(o11y_k8s_agent_yaml) def make_tidb_files(region, tenant_id, project_id, cluster_id): vpc = get_seed_vpc(region, project_id) admin_nodegroup = get_seed_admin_nodegroup( region, project_id, vpc["metadata"]["name"] ) private_endpoint = get_o11y_private_endpoint(region, tenant_id, project_id) tidb_nodegroup_yaml = gen_tidb_nodegroup( region=region, cluster_id=cluster_id, k8s_cluster_name=admin_nodegroup["spec"]["forProvider"]["clusterName"], node_role=admin_nodegroup["spec"]["forProvider"]["nodeRole"], subnet_ids=[vpc["status"]["nodeSubnets"][0]["subnetId"]], version=admin_nodegroup["spec"]["forProvider"]["version"], release_version=admin_nodegroup["spec"]["forProvider"]["releaseVersion"], provider_config_name=admin_nodegroup["spec"]["providerConfigRef"]["name"], ami_type=admin_nodegroup["spec"]["forProvider"]["amiType"], ) with open(f"tidb-nodegroup-{cluster_id}.yaml", "w") as f: f.write(tidb_nodegroup_yaml) o11y_tidb_agent_yaml = gen_o11y_tidb_agent( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, k8s_cluster_name=admin_nodegroup["spec"]["forProvider"]["clusterName"], account_id=private_endpoint["spec"]["aws"]["accountID"], vpc_id=private_endpoint["spec"]["aws"]["vpcID"], security_group_id=private_endpoint["spec"]["aws"]["securityGroupIDs"][0], node_role=admin_nodegroup["spec"]["forProvider"]["nodeRole"], node_subnet_id=vpc["status"]["nodeSubnets"][0]["subnetId"], node_subnet_ids=[item["subnetId"] for item in vpc["status"]["nodeSubnets"]], node_vpc_name=vpc["metadata"]["name"], node_zone=vpc["status"]["nodeSubnets"][0]["availabilityZone"], node_version=admin_nodegroup["spec"]["forProvider"]["version"], node_release_version=admin_nodegroup["spec"]["forProvider"]["releaseVersion"], provider_config_name=admin_nodegroup["spec"]["providerConfigRef"]["name"], node_ami_type=admin_nodegroup["spec"]["forProvider"]["amiType"], ) with open(f"tidb-agent-{cluster_id}.yaml", "w") as f: f.write(o11y_tidb_agent_yaml) def gen_k8s_apply_vpc_endpoint_and_node_group(region, project_id): print( f"kubectl --context {seed_ctx(region)} -n observability apply -f k8s-vpc-endpoint-{project_id}.yaml" ) print( f"kubectl --context {seed_ctx(region)} -n observability apply -f k8s-nodegroup-{project_id}.yaml" ) def gen_tidb_apply_node_group(region, cluster_id): print( f"kubectl --context {seed_ctx(region)} -n observability apply -f tidb-nodegroup-{cluster_id}.yaml" ) def gen_k8s_patch_o11y_cluster(region, tenant_id, project_id): k8s_cluster = get_o11y_cluster(region, tenant_id, project_id, "k8s-infra") print( f"""kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} patch cluster {k8s_cluster["metadata"]["name"]} --type='json' -p='[{{"op": "remove", "path": "/spec/agent"}}]'""" ) def gen_tidb_patch_o11y_cluster(region, tenant_id, project_id, cluster_id): tidb_cluster = get_o11y_cluster(region, tenant_id, project_id, cluster_id) print( f"""kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} patch cluster {tidb_cluster["metadata"]["name"]} --type='json' -p='[{{"op": "remove", "path": "/spec/agent"}}]'""" ) def gen_k8s_pause_o11yroute(region, project_id): appbox = get_seed_appbox(region, project_id) o11yroute_name = appbox["metadata"]["name"] print( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} annotate o11yroute {o11yroute_name} dedicated.regionalserver.tidbcloud.com/paused=true --overwrite" ) print( f"""kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} patch o11yroute {o11yroute_name} --type='json' -p='[{{"op": "remove", "path": "/metadata/finalizers"}}]'""" ) def gen_tidb_pause_o11yroute(region, cluster_id): print( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} annotate o11yroute {cluster_id} dedicated.regionalserver.tidbcloud.com/paused=true --overwrite" ) print( f"""kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} patch o11yroute {cluster_id} --type='json' -p='[{{"op": "remove", "path": "/metadata/finalizers"}}]'""" ) def gen_check_vmagent_buffer(cluster_id): pod = get_dataplane_vmagent_pod(cluster_id) pod_name = pod["metadata"]["name"] print( f"kubectl --context dedicated.{cluster_id} -n tidb{cluster_id} exec -it {pod_name} -- /bin/sh -c 'cat /tmpData/persistent-queue/*/metainfo.json'" ) def gen_k8s_migrate(region, tenant_id, project_id): agent = get_o11y_o11yagent(region, tenant_id, project_id, "k8s-infra") print( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} delete o11yagent {agent['metadata']['name']}" ) print( f"kubectl --context {seed_ctx(region)} -n observability apply -f k8s-agent-{project_id}.yaml" ) def gen_tidb_migrate(region, tenant_id, project_id, cluster_id): agent = get_o11y_o11yagent(region, tenant_id, project_id, cluster_id) print( f"kubectl --context {o11y_ctx(region)} -n tidbcloud-{tenant_id} delete o11yagent {agent['metadata']['name']}" ) print( f"kubectl --context {seed_ctx(region)} -n observability apply -f tidb-agent-{cluster_id}.yaml" ) def gen_k8s_delete_o11yroute(region, project_id): appbox = get_seed_appbox(region, project_id) o11yroute_name = appbox["metadata"]["name"] print( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} delete o11yroute {o11yroute_name}" ) def gen_tidb_delete_o11yroute(region, cluster_id): print( f"kubectl --context {seed_ctx(region)} -n dedicated-prod-{region} delete o11yroute {cluster_id}" ) def do_tenant(region, tenant_id): make_backup_files(region=region, tenant_id=tenant_id) def do_project(region, tenant_id, project_id): o11yagent = get_o11y_o11yagent( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id="k8s-infra", ) if not o11yagent: return make_k8s_files( region=region, tenant_id=tenant_id, project_id=project_id, ) print("# apply vpc endpoint and nodegroup") gen_k8s_apply_vpc_endpoint_and_node_group( region=region, project_id=project_id, ) print() print("# patch o11y cluster") gen_k8s_patch_o11y_cluster( region=region, tenant_id=tenant_id, project_id=project_id, ) print() print("# pause o11yroute") gen_k8s_pause_o11yroute( region=region, project_id=project_id, ) print() print("# migrate k8s") gen_k8s_migrate( region=region, tenant_id=tenant_id, project_id=project_id, ) print() print("# delete o11yroute") gen_k8s_delete_o11yroute( region=region, project_id=project_id, ) print() print(f"""# Wait for o11yk8sagent to become Normal echo "Waiting for o11yk8sagent {project_id} to become Normal..." while true; do PHASE=$(kubectl --context controlplane.seed-aws-{region} -n observability get o11yk8sagent {project_id} -o jsonpath='{{.status.phase}}' 2>/dev/null) if [ "$PHASE" = "Normal" ]; then echo "o11yk8sagent {project_id} is now Normal" break fi echo "Current phase: $PHASE, waiting 10 seconds..." sleep 10 done """) def do_cluster_1(region, tenant_id, project_id, cluster_id): make_tidb_files( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id ) print("# apply nodegroup") gen_tidb_apply_node_group( region=region, cluster_id=cluster_id, ) print() def do_cluster_2(region, tenant_id, project_id, cluster_id): print("# patch o11y cluster") gen_tidb_patch_o11y_cluster( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, ) print() print("# pause o11yroute") gen_tidb_pause_o11yroute( region=region, cluster_id=cluster_id, ) print() print("# migrate tidb") gen_tidb_migrate( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, ) print() print("# delete o11yroute") gen_tidb_delete_o11yroute( region=region, cluster_id=cluster_id, ) print() print(f"""# Wait for o11ytidbagent to become Normal echo "Waiting for o11ytidbagent {cluster_id} to become Normal..." while true; do PHASE=$(kubectl --context controlplane.seed-aws-{region} -n observability get o11ytidbagent {cluster_id} -o jsonpath='{{.status.phase}}' 2>/dev/null) if [ "$PHASE" = "Normal" ]; then echo "o11ytidbagent {cluster_id} is now Normal" break fi echo "Current phase: $PHASE, waiting 10 seconds..." sleep 10 done """) clusters_ap_northeast_1 = [ { "tenant_id": "1372813089208221493", "project_id": "1372813089206361379", "cluster_id": "1379661944638574071", "created_at": "2022-12-21 08:22:55", "name": "cpalearning", "status": "normal", }, { "tenant_id": "1372813089190911285", "project_id": "1372813089206751291", "cluster_id": "1379661944638454106", "created_at": "2022-12-21 01:59:26", "name": "CTW Inc.", "status": "normal", }, { "tenant_id": "1372813089199011282", "project_id": "1372813089206481394", # standalone "cluster_id": "1379661944636084123", "created_at": "2022-11-21 01:27:30", "name": "ELESTYLE", "status": "normal", }, { "tenant_id": "1372813089208611456", "project_id": "1372813089310221297", "cluster_id": "1379661944644514084", "created_at": "2023-03-24 02:52:55", "name": "Mico Inc.", "status": "normal", }, { "tenant_id": "1372813089208611456", "project_id": "1372813089318591287", "cluster_id": "1379661944644934074", "created_at": "2023-03-29 07:03:07", "name": "Mico Inc.", "status": "paused", }, { "tenant_id": "1372813089190491281", "project_id": "1372813089206331353", "cluster_id": "1379661944645264080", "created_at": "2023-04-03 10:24:48", "name": "istyle Inc.", "status": "normal", }, { "tenant_id": "1372813089208611456", "project_id": "1372813089310251285", "cluster_id": "1379661944646195319", "created_at": "2023-05-08 03:12:30", "name": "Mico Inc.", "status": "normal", }, { "tenant_id": "1372813089195441286", "project_id": "1372813089374841286", "cluster_id": "1379661944646195886", "created_at": "2023-05-17 06:21:31", "name": "DMM", "status": "normal", }, { "tenant_id": "1372813089191271282", "project_id": "1372813089206331321", "cluster_id": "1379661944646167822", "created_at": "2023-06-15 02:40:34", "name": "SB Payment Service", "status": "normal", }, { "tenant_id": "1372813089208251343", "project_id": "1372813089364551285", "cluster_id": "1379661944646255118", "created_at": "2023-07-10 00:49:06", "name": "RHEMS Japan", "status": "normal", }, { "tenant_id": "1372813089209001464", "project_id": "1372813089434541283", "cluster_id": "1379661944646225237", "created_at": "2023-07-10 11:22:55", "name": "Fujitsu Japan", "status": "normal", }, { "tenant_id": "1372813089209001464", "project_id": "1372813089435591287", "cluster_id": "1379661944646225238", "created_at": "2023-07-10 11:24:21", "name": "Fujitsu Japan", "status": "normal", }, { "tenant_id": "1372813089209001464", "project_id": "1372813089435591288", "cluster_id": "1379661944646255177", "created_at": "2023-07-10 11:25:09", "name": "Fujitsu Japan", "status": "normal", }, { "tenant_id": "1372813089209001464", "project_id": "1372813089435591289", "cluster_id": "1379661944646225239", "created_at": "2023-07-10 11:25:56", "name": "Fujitsu Japan", "status": "normal", }, { "tenant_id": "1372813089209001464", "project_id": "1372813089435591290", "cluster_id": "1379661944646255178", "created_at": "2023-07-10 11:26:49", "name": "Fujitsu Japan", "status": "paused", }, { "tenant_id": "1372813089191601283", "project_id": "1372813089454312732", "cluster_id": "1379661944646258848", "created_at": "2023-08-02 14:24:54", "name": "SB Payment Service(not poc)", "status": "normal", }, { "tenant_id": "1372813089191601283", "project_id": "1372813089454312735", "cluster_id": "1379661944646229002", "created_at": "2023-08-02 15:07:22", "name": "SB Payment Service(not poc)", "status": "normal", }, { "tenant_id": "1372813089195441286", "project_id": "1372813089454312041", "cluster_id": "1379661944646229300", "created_at": "2023-08-04 05:06:19", "name": "DMM", "status": "normal", }, { "tenant_id": "1372813089190491281", "project_id": "1372813089206331293", "cluster_id": "1379661944646230150", "created_at": "2023-08-08 13:10:16", "name": "istyle Inc.", "status": "normal", }, { "tenant_id": "1372813089209062317", "project_id": "1372813089454401282", "cluster_id": "1379661944646314069", "created_at": "2023-08-17 06:17:23", "name": "BNE-BSR", "status": "normal", }, { "tenant_id": "1372813089208341734", "project_id": "1372813089206841293", "cluster_id": "1379661944646344141", "created_at": "2023-08-28 08:08:50", "name": "Sparty", "status": "normal", }, { "tenant_id": "1372813089190491281", "project_id": "1372813089206301308", "cluster_id": "1379661944646412764", "created_at": "2023-10-18 01:06:37", "name": "istyle Inc.", "status": "normal", }, { "tenant_id": "1372813089190911285", "project_id": "1372813089206331466", "cluster_id": "1379661944646413464", "created_at": "2023-12-14 05:11:59", "name": "CTW Inc.", "status": "normal", }, { "tenant_id": "1372813089190911285", "project_id": "1372813089190071289", "cluster_id": "1379661944646413522", "created_at": "2023-12-21 01:29:46", "name": "CTW Inc.", "status": "normal", }, { "tenant_id": "1372813089209214865", "project_id": "1372813089454532367", "cluster_id": "1379661944646413605", "created_at": "2024-01-08 02:58:44", "name": "bitget wallet", "status": "normal", }, { "tenant_id": "1372813089209212394", "project_id": "1372813089454533532", "cluster_id": "1379661944646413662", "created_at": "2024-01-10 09:10:19", "name": "dataX", "status": "paused", }, { "tenant_id": "1372813089209212394", "project_id": "1372813089454533531", "cluster_id": "1379661944646413663", "created_at": "2024-01-10 09:12:44", "name": "dataX", "status": "normal", }, { "tenant_id": "1372813089209212842", "project_id": "1372813089454524730", "cluster_id": "1379661944646413694", "created_at": "2024-01-12 11:04:20", "name": "Orb Inc.", "status": "normal", }, { "tenant_id": "1372813089209216306", "project_id": "1372813089454534527", "cluster_id": "1379661944646413836", "created_at": "2024-01-24 02:46:02", "name": "CAPCOM-MP-TiDB-REBE-USER-TEST", "status": "paused", }, { "tenant_id": "1372813089209219376", "project_id": "1372813089454535088", "cluster_id": "1379661944646413967", "created_at": "2024-02-09 04:38:02", "name": "CAPCOM-CM-TIDB-REBE-USER-COMMON", "status": "paused", }, { "tenant_id": "1372813089208521826", "project_id": "1372813089454524672", "cluster_id": "1379661944646413981", "created_at": "2024-02-14 02:35:33", "name": "voicy", "status": "normal", }, { "tenant_id": "1372813089209219376", "project_id": "1372813089454535088", "cluster_id": "1379661944646414280", "created_at": "2024-03-18 02:51:50", "name": "CAPCOM-CM-TIDB-REBE-USER-COMMON", "status": "paused", }, { "tenant_id": "1372813089209218394", "project_id": "1372813089454539809", "cluster_id": "1379661944646414487", "created_at": "2024-04-04 10:38:30", "name": "Levtech", "status": "normal", }, { "tenant_id": "1372813089209219376", "project_id": "1372813089454535088", "cluster_id": "1379661944646414580", "created_at": "2024-04-16 07:18:37", "name": "CAPCOM-CM-TIDB-REBE-USER-COMMON", "status": "paused", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454542218", "cluster_id": "1379661944646414631", "created_at": "2024-04-18 02:56:15", "name": "SUPER STUDIO", "status": "normal", }, { "tenant_id": "1372813089209225520", "project_id": "1372813089454542195", "cluster_id": "1379661944646414633", "created_at": "2024-04-18 07:16:32", "name": "CAPCOM-CM-TIDB-REBE-USER-TESTBED", "status": "paused", }, { "tenant_id": "1372813089209225520", "project_id": "1372813089454542195", "cluster_id": "1379661944646414634", "created_at": "2024-04-18 07:16:33", "name": "CAPCOM-CM-TIDB-REBE-USER-TESTBED", "status": "normal", }, { "tenant_id": "1372813089208521826", "project_id": "1372813089454524673", # standalone "cluster_id": "1379661944646414638", "created_at": "2024-04-19 02:55:19", "name": "voicy", "status": "normal", }, { "tenant_id": "1372813089208521826", "project_id": "1372813089454343828", "cluster_id": "1379661944646414690", "created_at": "2024-04-25 04:43:46", "name": "voicy", "status": "normal", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454542218", "cluster_id": "1379661944646414743", "created_at": "2024-05-04 14:38:37", "name": "SUPER STUDIO", "status": "paused", }, { "tenant_id": "1372813089209228506", "project_id": "1372813089454545428", "cluster_id": "1379661944646414817", "created_at": "2024-05-15 05:53:31", "name": "CAPCOM-CM-TiDB-REBE-USER-STR", "status": "paused", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454542217", # standalone "cluster_id": "1379661944646414964", "created_at": "2024-05-27 03:11:08", "name": "SUPER STUDIO", "status": "normal", }, { "tenant_id": "1372813089209062317", "project_id": "1372813089454547307", "cluster_id": "1379661944646415021", "created_at": "2024-05-29 02:54:23", "name": "BNE-BSR", "status": "normal", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454548011", "cluster_id": "1379661944646415112", "created_at": "2024-06-06 23:37:39", "name": "SUPER STUDIO", "status": "normal", }, { "tenant_id": "1372813089209218394", "project_id": "1372813089454539808", # standalone "cluster_id": "1379661944646415416", "created_at": "2024-07-11 03:00:17", "name": "Levtech", "status": "normal", }, { "tenant_id": "1372813089209219376", "project_id": "1372813089454535088", "cluster_id": "1379661944646415597", "created_at": "2024-08-09 08:22:09", "name": "CAPCOM-CM-TIDB-REBE-USER-COMMON", "status": "normal", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454555231", "cluster_id": "1379661944646415623", "created_at": "2024-08-14 02:34:17", "name": "SUPER STUDIO", "status": "normal", }, { "tenant_id": "1372813089209228505", "project_id": "1372813089454549249", "cluster_id": "1379661944646415736", "created_at": "2024-08-26 09:09:30", "name": "CAPCOM-CM-TiDB-REBE-USER-PRD", "status": "normal", }, { "tenant_id": "1372813089209228505", "project_id": "1372813089454549249", "cluster_id": "1379661944646415737", "created_at": "2024-08-26 09:09:30", "name": "CAPCOM-CM-TiDB-REBE-USER-PRD", "status": "normal", }, { "tenant_id": "1372813089209216864", "project_id": "1372813089454558801", "cluster_id": "1379661944646415890", "created_at": "2024-09-13 04:49:15", "name": "CAPCOM CO LTD", "status": "normal", }, { "tenant_id": "1372813089209212842", "project_id": "1372813089454559969", "cluster_id": "1379661944646415950", "created_at": "2024-09-26 15:44:41", "name": "Orb Inc.", "status": "paused", }, { "tenant_id": "1372813089209238544", "project_id": "1372813089454560618", "cluster_id": "10662888935658790607", "created_at": "2024-10-04 04:51:09", "name": "TOKYO GAS", "status": "paused", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454547722", "cluster_id": "1379661944646416090", "created_at": "2024-10-25 02:19:20", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089209238544", "project_id": "1372813089454563820", "cluster_id": "10901200605387644060", "created_at": "2024-10-31 13:50:12", "name": "TOKYO GAS", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454547722", "cluster_id": "1379661944646416142", "created_at": "2024-11-05 06:22:37", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454542218", "cluster_id": "1379661944646416207", "created_at": "2024-11-11 07:11:40", "name": "SUPER STUDIO", "status": "paused", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454552682", "cluster_id": "1379661944646416286", "created_at": "2024-11-20 08:12:15", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454565453", "cluster_id": "1379661944646416287", "created_at": "2024-11-20 08:14:29", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089209230886", "project_id": "1372813089454548064", "cluster_id": "10705947703365610402", "created_at": "2024-12-02 06:30:07", "name": "Future", "status": "normal", }, { "tenant_id": "1372813089209247532", "project_id": "1372813089454565273", "cluster_id": "10306304876687101386", "created_at": "2024-12-04 07:47:47", "name": "mynavi_cm_furusato", "status": "normal", }, # { # "tenant_id": "1372813089187791289", # "project_id": "1372813089454540451", # "cluster_id": "10962092174136419751", # "created_at": "2024-12-05 02:26:57", # "name": "dbaastest3", # "status": "normal", # }, { "tenant_id": "1372813089209247520", "project_id": "1372813089454565856", "cluster_id": "10185497361830705734", "created_at": "2024-12-23 02:45:19", "name": "JALUX Inc.", "status": "normal", }, { "tenant_id": "1372813089209252089", "project_id": "1372813089454569768", "cluster_id": "10100520101539780704", "created_at": "2024-12-27 01:12:36", "name": "BNE-EARTH", "status": "normal", }, { "tenant_id": "1372813089209249748", "project_id": "1372813089454570991", "cluster_id": "10116563278625933647", "created_at": "2025-01-07 02:23:32", "name": "ap-rap", "status": "normal", }, { "tenant_id": "1372813089209249748", "project_id": "1372813089454568094", "cluster_id": "10821441401704225062", "created_at": "2025-01-08 07:29:34", "name": "ap-rap", "status": "normal", }, { "tenant_id": "1372813089209241995", "project_id": "1372813089454559589", "cluster_id": "10386986038834188822", "created_at": "2025-01-29 08:13:29", "name": "baobab game studios", "status": "normal", }, { "tenant_id": "1372813089209240001", "project_id": "1372813089454557588", "cluster_id": "10751045358657452884", "created_at": "2025-02-06 05:18:49", "name": "COLLABO STYLE", "status": "normal", }, { "tenant_id": "1372813089209212394", "project_id": "1372813089454533532", "cluster_id": "1379661944646416539", "created_at": "2025-02-13 05:35:31", "name": "dataX", "status": "paused", }, { "tenant_id": "1372813089209212394", "project_id": "1372813089454533531", "cluster_id": "1379661944646416541", "created_at": "2025-02-14 10:26:33", "name": "dataX", "status": "normal", }, { "tenant_id": "1372813089209248707", "project_id": "1372813089454574517", "cluster_id": "10864749137309062290", "created_at": "2025-02-19 07:53:02", "name": "Andrew's Org", "status": "normal", }, { "tenant_id": "1372813089209031895", "project_id": "1372813089454542217", # standalone "cluster_id": "1379661944646416563", "created_at": "2025-02-21 05:48:35", "name": "SUPER STUDIO", "status": "normal", }, { "tenant_id": "1372813089209256793", "project_id": "1372813089454574660", # standalone "cluster_id": "10601809957776321988", "created_at": "2025-02-21 08:13:32", "name": "Booboo's Org", "status": "normal", }, { "tenant_id": "1372813089194211283", "project_id": "1372813089454343321", "cluster_id": "10354969479543878958", "created_at": "2025-02-25 19:56:12", "name": "PingCAP_Japan", "status": "paused", }, { "tenant_id": "1372813089209248707", "project_id": "1372813089454575106", "cluster_id": "10654218712045887222", "created_at": "2025-02-27 07:01:54", "name": "Andrew's Org", "status": "normal", }, { "tenant_id": "1372813089190911285", "project_id": "1372813089454574422", "cluster_id": "10396262662770131016", "created_at": "2025-02-28 07:17:11", "name": "CTW Inc.", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575118", "cluster_id": "10016617196374558903", "created_at": "2025-03-03 07:31:11", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089209257781", "project_id": "1372813089454575550", "cluster_id": "10382792117723215762", "created_at": "2025-03-17 05:00:22", "name": "bitget", "status": "normal", }, { "tenant_id": "1372813089209255759", "project_id": "1372813089454575407", "cluster_id": "10154965683029345713", "created_at": "2025-03-18 07:22:06", "name": "Infcurion", "status": "normal", }, { "tenant_id": "1372813089209224877", "project_id": "1372813089454541916", "cluster_id": "10454342977024998915", "created_at": "2025-03-18 08:14:15", "name": "BNE-Jumpplus10th", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10435271782269042237", "created_at": "2025-03-28 04:05:39", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10266703332935765966", "created_at": "2025-03-28 04:05:39", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10961550154398712894", "created_at": "2025-03-28 04:05:39", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10258494454765409384", "created_at": "2025-03-28 04:05:39", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10128663844535194068", "created_at": "2025-03-28 04:05:39", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10961033529265313143", "created_at": "2025-03-31 09:11:55", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089209213383", "project_id": "1372813089454525893", "cluster_id": "10594541336353152449", "created_at": "2025-04-07 10:39:03", "name": "TIS Inc.", "status": "normal", }, { "tenant_id": "1372813089209212394", "project_id": "1372813089454579695", "cluster_id": "10113865644681298064", "created_at": "2025-04-22 01:52:12", "name": "dataX", "status": "normal", }, { "tenant_id": "1372813089209256793", "project_id": "1372813089454574660", # standalone "cluster_id": "10129313401127841163", "created_at": "2025-05-06 10:33:20", "name": "Booboo's Org", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454549134", "cluster_id": "10646585945675031829", "created_at": "2025-05-07 05:57:56", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10264555407364369315", "created_at": "2025-05-13 03:00:51", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454539049", "cluster_id": "10947714666900611238", "created_at": "2025-05-16 06:45:51", "name": "Cygames", "status": "normal", }, { "tenant_id": "1372813089209256793", "project_id": "1372813089454576996", # standalone "cluster_id": "10970866242657397869", "created_at": "2025-05-16 08:03:42", "name": "Booboo's Org", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454566814", "cluster_id": "10371861082285997722", "created_at": "2025-05-26 06:40:16", "name": "Cygames", "status": "paused", }, { "tenant_id": "1372813089209238544", "project_id": "1372813089454556123", "cluster_id": "10526477560423163174", "created_at": "2025-05-30 02:34:00", "name": "TOKYO GAS", "status": "paused", }, { "tenant_id": "1372813089209256793", "project_id": "1372813089454576996", # standalone "cluster_id": "10427561086145496095", "created_at": "2025-06-03 05:50:43", "name": "Booboo's Org", "status": "normal", }, { "tenant_id": "1372813089209267920", "project_id": "1372813089454585908", "cluster_id": "10117946960366138983", "created_at": "2025-06-19 10:06:33", "name": "ponyft-aws", "status": "normal", }, { "tenant_id": "1372813089209255759", "project_id": "1372813089454575407", "cluster_id": "10773171859209315262", "created_at": "2025-07-03 06:46:51", "name": "Infcurion", "status": "paused", }, { "tenant_id": "1372813089209254049", "project_id": "1372813089454571729", "cluster_id": "10169657109889742509", "created_at": "2025-07-07 00:05:05", "name": "VideoResearch Ltd", "status": "normal", }, { "tenant_id": "1372813089209215669", "project_id": "1372813089454581193", "cluster_id": "10662531262795861512", "created_at": "2025-07-07 11:14:52", "name": "coincheck", "status": "normal", }, { "tenant_id": "1372813089209267920", "project_id": "1372813089454585670", "cluster_id": "10193654782171099315", "created_at": "2025-07-10 01:23:40", "name": "ponyft-aws", "status": "normal", }, { "tenant_id": "1372813089209266501", "project_id": "1372813089454588228", "cluster_id": "10954498571155998794", "created_at": "2025-07-10 04:24:11", "name": "Suntory Holdings Limited_Information Technology Department_B2C", "status": "normal", }, { "tenant_id": "1372813089209251423", "project_id": "1372813089454569094", "cluster_id": "10431654850386443887", "created_at": "2025-07-11 08:05:05", "name": "Osakagas", "status": "normal", }, { "tenant_id": "1372813089209224877", "project_id": "1372813089454588873", "cluster_id": "10024315649877557596", "created_at": "2025-07-16 02:36:31", "name": "BNE-Jumpplus10th", "status": "normal", }, { "tenant_id": "1372813089209215669", "project_id": "1372813089454529561", "cluster_id": "10765680971272299657", "created_at": "2025-07-24 12:21:10", "name": "coincheck", "status": "normal", }, { "tenant_id": "1372813089209215669", "project_id": "1372813089454529561", "cluster_id": "10332566432948937427", "created_at": "2025-07-30 05:25:35", "name": "coincheck", "status": "normal", }, { "tenant_id": "1372813089209273112", "project_id": "1372813089454590922", "cluster_id": "10598809482713269480", "created_at": "2025-07-31 04:56:59", "name": "system's Org", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454549134", "cluster_id": "10842527965119794080", "created_at": "2025-08-19 01:01:13", "name": "Cygames", "status": "paused", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454549134", "cluster_id": "10019139233925469623", "created_at": "2025-08-19 02:13:21", "name": "Cygames", "status": "paused", }, { "tenant_id": "1372813089209263571", "project_id": "1372813089454594055", "cluster_id": "10623328384120554982", "created_at": "2025-08-26 23:05:02", "name": "aiinside", "status": "normal", }, { "tenant_id": "1372813089209265136", "project_id": "1372813089454582860", "cluster_id": "10450972314169086379", "created_at": "2025-08-27 04:52:37", "name": "Safie Staging Org", "status": "paused", }, { "tenant_id": "1372813089209032432", "project_id": "1372813089454574946", # standalone "cluster_id": "10603657072080039256", "created_at": "2025-08-29 03:45:22", "name": "gumi-payer", "status": "normal", }, { "tenant_id": "1372813089208611456", "project_id": "1372813089310221297", "cluster_id": "10947113757820990962", "created_at": "2025-09-04 04:12:41", "name": "Mico Inc.", "status": "normal", }, { "tenant_id": "1372813089191061283", "project_id": "1372813089454575516", "cluster_id": "10244862203872145352", "created_at": "2025-09-16 09:30:32", "name": "Cygames", "status": "normal", }, # { # "tenant_id": "1372813089209281518", # "project_id": "1372813089454599400", # "cluster_id": "10668245128865083302", # "created_at": "2025-10-13 17:21:55", # "name": "lin's Org", # "status": "normal", # }, # { # "tenant_id": "1372813089191061283", # "project_id": "1372813089454575516", # "cluster_id": "10902950268352154472", # "created_at": "2025-10-16 09:24:07", # "name": "Cygames", # "status": "normal", # }, # { # "tenant_id": "1372813089209279897", # "project_id": "1372813089454598302", # "cluster_id": "10775187987829920573", # "created_at": "2025-10-17 01:52:37", # "name": "SonyMusicSolutions", # "status": "normal", # }, # { # "tenant_id": "1372813089209215669", # "project_id": "1372813089454529561", # "cluster_id": "10694344419194927933", # "created_at": "2025-10-21 01:26:54", # "name": "coincheck", # "status": "normal", # }, # { # "tenant_id": "1372813089209252085", # "project_id": "1372813089454600673", # "cluster_id": "10398776417319404572", # "created_at": "2025-10-22 03:50:40", # "name": "BNE-Fast", # "status": "normal", # }, # { # "tenant_id": "1372813089209032432", # "project_id": "1372813089454526409", # "cluster_id": "10465034408831849080", # "created_at": "2025-10-24 04:46:18", # "name": "gumi-payer", # "status": "paused", # }, # { # "tenant_id": "1372813089209254049", # "project_id": "1372813089454571729", # "cluster_id": "10270195332516769545", # "created_at": "2025-11-05 01:50:55", # "name": "VideoResearch Ltd", # "status": "normal", # }, # { # "tenant_id": "1372813089209213383", # "project_id": "1372813089454525893", # "cluster_id": "10908918689344012181", # "created_at": "2025-11-06 07:51:17", # "name": "TIS Inc.", # "status": "normal", # }, # { # "tenant_id": "1372813089191061283", # "project_id": "1372813089454575516", # "cluster_id": "10749207329779541820", # "created_at": "2025-11-12 03:35:21", # "name": "Cygames", # "status": "normal", # }, # { # "tenant_id": "1372813089209282380", # "project_id": "1372813089454600272", # "cluster_id": "10641520066088958688", # "created_at": "2025-11-14 03:31:25", # "name": "kubell", # "status": "normal", # }, # { # "tenant_id": "1372813089209265136", # "project_id": "1372813089454605301", # "cluster_id": "10758616352933390728", # "created_at": "2025-11-17 04:50:56", # "name": "Safie Staging Org", # "status": "paused", # }, # { # "tenant_id": "1372813089208611456", # "project_id": "1372813089454541062", # "cluster_id": "10391156492575830228", # "created_at": "2025-11-17 07:40:41", # "name": "Mico Inc.", # "status": "normal", # }, # { # "tenant_id": "1372813089209212842", # "project_id": "1372813089454559969", # "cluster_id": "10297123031077348333", # "created_at": "2025-11-18 02:13:21", # "name": "Orb Inc.", # "status": "paused", # }, # { # "tenant_id": "1372813089209266501", # "project_id": "1372813089454604333", # "cluster_id": "10037176470437812922", # "created_at": "2025-11-19 06:30:50", # "name": "Suntory Holdings Limited_Information Technology Department_B2C", # "status": "normal", # }, # { # "tenant_id": "1372813089209214865", # "project_id": "1372813089454532367", # "cluster_id": "10135821339326849649", # "created_at": "2025-11-24 09:46:50", # "name": "bitget wallet", # "status": "normal", # }, # { # "tenant_id": "1372813089209219376", # "project_id": "1372813089454535088", # "cluster_id": "10732272960853787309", # "created_at": "2025-11-28 08:39:48", # "name": "CAPCOM-CM-TIDB-REBE-USER-COMMON", # "status": "paused", # }, # { # "tenant_id": "1372813089209219376", # "project_id": "1372813089454535088", # "cluster_id": "10163463667478777848", # "created_at": "2025-11-28 08:40:49", # "name": "CAPCOM-CM-TIDB-REBE-USER-COMMON", # "status": "paused", # }, # { # "tenant_id": "1372813089209255759", # "project_id": "1372813089454603146", # "cluster_id": "10417312570643048991", # "created_at": "2025-12-16 07:46:45", # "name": "Infcurion", # "status": "paused", # }, # { # "tenant_id": "1372813089208611456", # "project_id": "1372813089310251285", # "cluster_id": "10955158406804822786", # "created_at": "2025-12-16 09:53:50", # "name": "Mico Inc.", # "status": "paused", # }, # { # "tenant_id": "1372813089209215669", # "project_id": "1372813089454529561", # "cluster_id": "10865349947122674119", # "created_at": "2025-12-17 08:13:56", # "name": "coincheck", # "status": "paused", # }, # { # "tenant_id": "1372813089208611456", # "project_id": "1372813089318591287", # "cluster_id": "10058061063842538906", # "created_at": "2025-12-23 10:02:58", # "name": "Mico Inc.", # "status": "paused", # }, # { # "tenant_id": "1372813089209031895", # "project_id": "1372813089454542218", # "cluster_id": "10036394790812148955", # "created_at": "2026-01-05 05:12:53", # "name": "SUPER STUDIO", # "status": "normal", # }, # { # "tenant_id": "1372813089188691290", # "project_id": "1372813089454576539", # "cluster_id": "10084264493313187252", # "created_at": "2026-01-07 03:00:29", # "name": "TiDB Cluster Service", # "status": "normal", # }, # { # "tenant_id": "1372813089209212842", # "project_id": "1372813089454524730", # "cluster_id": "10476187478882994208", # "created_at": "2026-01-07 05:49:10", # "name": "Orb Inc.", # "status": "normal", # }, # { # "tenant_id": "1372813089209265136", # "project_id": "1372813089454605301", # "cluster_id": "10519304419164184700", # "created_at": "2026-01-07 07:57:47", # "name": "Safie Staging Org", # "status": "paused", # }, # { # "tenant_id": "1372813089194211283", # "project_id": "1372813089454568907", # "cluster_id": "10270708215133263144", # "created_at": "2026-01-07 12:23:34", # "name": "PingCAP_Japan", # "status": "paused", # }, # { # "tenant_id": "1372813089209001464", # "project_id": "1372813089435591287", # "cluster_id": "10400189636047870733", # "created_at": "2026-01-08 03:08:31", # "name": "Fujitsu Japan", # "status": "normal", # }, # { # "tenant_id": "1372813089209293818", # "project_id": "1372813089454613420", # "cluster_id": "10567149411922876303", # "created_at": "2026-01-08 06:54:05", # "name": "PROD-KIKAN-DB", # "status": "paused", # }, # { # "tenant_id": "1372813089209283835", # "project_id": "1372813089454613940", # "cluster_id": "10176320693621335567", # "created_at": "2026-01-11 16:08:24", # "name": "f4samurai_revenge_prd", # "status": "normal", # }, # { # "tenant_id": "1372813089194211283", # "project_id": "1372813089454575766", # "cluster_id": "10143359564711019416", # "created_at": "2026-01-13 00:30:02", # "name": "PingCAP_Japan", # "status": "normal", # }, # { # "tenant_id": "1372813089191061283", # "project_id": "1372813089454539049", # "cluster_id": "10717054772700266074", # "created_at": "2026-01-13 02:37:26", # "name": "Cygames", # "status": "paused", # }, # { # "tenant_id": "1372813089209215669", # "project_id": "1372813089454614501", # "cluster_id": "10582611283601734430", # "created_at": "2026-01-13 09:52:53", # "name": "coincheck", # "status": "normal", # }, # { # "tenant_id": "1372813089195441286", # "project_id": "1372813089374841286", # "cluster_id": "10060208976988389031", # "created_at": "2026-01-14 03:39:42", # "name": "DMM", # "status": "recovering", # }, ] if __name__ == "__main__": region = "ap-northeast-1" # tenants = set() # projects = set() # for c in clusters_ap_northeast_1: # tenants.add(c["tenant_id"]) # projects.add((c["tenant_id"], c["project_id"])) # for tenant_id in tenants: # do_tenant( # region=region, # tenant_id=tenant_id, # ) # for tenant_id, project_id in projects: # print("# Project:", project_id) # print() # do_project( # region=region, # tenant_id=tenant_id, # project_id=project_id, # ) # print() do_project( region=region, tenant_id="1372813089208611456", project_id="1372813089454541062", ) print() # print() # print("===") # print() # for c in clusters_ap_northeast_1: # print("# Cluster:", c["cluster_id"]) # print() # do_cluster_2( # region=region, # tenant_id=c["tenant_id"], # project_id=c["project_id"], # cluster_id=c["cluster_id"], # ) # print()