#!/usr/bin/env python3 import json import subprocess def seed_ctx(region): return f"controlplane.seed-gcp-{region}" def o11y_ctx(region): return f"controlplane.o11y-prod-regional-{region}" def seed_ns(region): return f"dedicated-prod-{region}" def o11y_ns(tenant_id): return f"tidbcloud-{tenant_id}" def run(command: str): print("# [Running]:", command) return run_raw(command.split()) def run_raw(command): try: return subprocess.run(command, capture_output=True, text=True, check=True) except subprocess.CalledProcessError as e: print(f"Error: {e}") print(f"stdout: {e.stdout}") print(f"stderr: {e.stderr}") raise def get_seed_vpc(region, project_id): result = run( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} get vpc -l tags.tidbcloud.com/project={project_id} -o json" ) vpcs = json.loads(result.stdout) if len(vpcs["items"]) > 1: raise ValueError("Multiple vpcs found") elif len(vpcs["items"]) == 0: raise ValueError("No vpcs found") return vpcs["items"][0] def get_seed_appbox(region, project_id): result = run( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} get appboxes -l tags.tidbcloud.com/project={project_id} -o json" ) appboxes = json.loads(result.stdout) if len(appboxes["items"]) > 1: raise ValueError("Multiple appboxes found") elif len(appboxes["items"]) == 0: raise ValueError("No appboxes found") return appboxes["items"][0] def get_seed_nodepool(region, project_id, vpc_name): result = run( f"kubectl --context {seed_ctx(region)} get nodepools.container.gcp.crossplane.io -l tidbcloud-network={vpc_name} -o json" ) nodepools = json.loads(result.stdout) for item in nodepools["items"]: if item["metadata"]["name"].startswith("tidbcloud-dedicated-"): return item result = run( f"kubectl --context {seed_ctx(region)} get nodepools.container.gcp.crossplane.io -l tidbcloud-namespace=prod-{project_id} -o json" ) nodepools = json.loads(result.stdout) for item in nodepools["items"]: if item["metadata"]["name"].startswith(f"tidbcloud-prod-{project_id}"): return item raise ValueError("Admin nodepool not found") def get_o11y_private_endpoint(region, tenant_id, project_id): result = run( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} get privateendpoint -l observability.pingcap.com/projectID={project_id} -o json" ) private_endpoints = json.loads(result.stdout) # if len(private_endpoints["items"]) > 1: # raise ValueError("Multiple private endpoints found") if len(private_endpoints["items"]) == 0: raise ValueError("No private endpoints found") return private_endpoints["items"][0] def get_o11y_cluster(region, tenant_id, project_id, cluster_id): result = run( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} get cluster -l observability.pingcap.com/applicationID={cluster_id},observability.pingcap.com/projectID={project_id} -o json" ) clusters = json.loads(result.stdout) if len(clusters["items"]) == 0: print(f"No cluster found with {project_id}/{cluster_id}") return elif len(clusters["items"]) > 1: print(f"Multiple clusters found with {project_id}/{cluster_id}") return return clusters["items"][0] def get_o11y_o11yagent(region, tenant_id, project_id, cluster_id): result = run( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} get o11yagent -l tags.tidbcloud.com/cluster={cluster_id},tags.tidbcloud.com/project={project_id} -o json" ) agents = json.loads(result.stdout) if len(agents["items"]) == 0: print(f"No cluster found with {project_id}/{cluster_id}") return elif len(agents["items"]) > 1: print(f"Multiple clusters found with {project_id}/{cluster_id}") return return agents["items"][0] def get_dataplane_vmagent_pod(cluster_id): result = run( f"kubectl --context dedicated.{cluster_id} -n tidb{cluster_id} get pod -l app.kubernetes.io/name=victoria-metrics-agent -o json" ) pods = json.loads(result.stdout) if len(pods["items"]) == 0: print(f"No vmagent pod found with {cluster_id}") return elif len(pods["items"]) > 1: print(f"Multiple vmagent pod found with {cluster_id}") return return pods["items"][0] def gen_gke_k8s_nodepool( region, tenant_id, project_id, appbox, network, cluster, cluster_ref, service_account, locations, version, provider_config_name, env="prod", machine_type="n2-standard-2", disk_size_gb=100, min_node_count=1, max_node_count=3, ): locations_json = json.dumps(locations) return f""" apiVersion: container.gcp.crossplane.io/v1beta1 kind: NodePool metadata: labels: o11y.tidbcloud.com/component: o11y o11y.tidbcloud.com/project: "{project_id}" name: tidbcloud-o11y-k8s-{project_id} spec: deletionPolicy: Delete forProvider: autoscaling: enabled: true maxNodeCount: {max_node_count} minNodeCount: {min_node_count} cluster: "{cluster}" clusterRef: name: "{cluster_ref}" config: diskSizeGb: {disk_size_gb} diskType: pd-standard imageType: cos_containerd labels: Name: tidbcloud-o11y-k8s-{project_id} environment: {env} project: "{project_id}" servicetype: dedicated tenant: "{tenant_id}" usedby: o11y o11y.tidbcloud.com/component: o11y o11y.tidbcloud.com/project: "{project_id}" tidbcloud.com/nodegroup-name: tidbcloud-o11y-k8s-{project_id} machineType: {machine_type} metadata: disable-legacy-endpoints: "true" oauthScopes: - https://www.googleapis.com/auth/cloud-platform serviceAccount: {service_account} taints: - effect: NO_EXECUTE key: o11y-dedicated value: o11y - effect: NO_EXECUTE key: o11y-project value: "{project_id}" shieldedInstanceConfig: enableIntegrityMonitoring: true workloadMetadataConfig: mode: GKE_METADATA initialNodeCount: 1 locations: {locations_json} management: autoRepair: false maxPodsConstraint: maxPodsPerNode: 32 upgradeSettings: maxSurge: 1 version: {version} providerConfigRef: name: {provider_config_name} """ def gen_gke_tidb_nodepool( region, tenant_id, project_id, cluster_id, appbox, network, cluster, cluster_ref, service_account, locations, version, provider_config_name, env="prod", machine_type="n2-standard-2", disk_size_gb=100, min_node_count=1, max_node_count=3, ): locations_json = json.dumps(locations) return f""" apiVersion: container.gcp.crossplane.io/v1beta1 kind: NodePool metadata: labels: o11y.tidbcloud.com/component: o11y o11y.tidbcloud.com/project: "{project_id}" o11y.tidbcloud.com/cluster: "{cluster_id}" name: tidbcloud-o11y-tidb-{cluster_id} spec: deletionPolicy: Delete forProvider: autoscaling: enabled: true maxNodeCount: {max_node_count} minNodeCount: {min_node_count} cluster: "{cluster}" clusterRef: name: "{cluster_ref}" config: diskSizeGb: {disk_size_gb} diskType: pd-standard imageType: cos_containerd labels: Name: tidbcloud-o11y-tidb-{cluster_id} cluster: "{cluster_id}" environment: {env} project: "{project_id}" servicetype: dedicated tenant: "{tenant_id}" usedby: o11y o11y.tidbcloud.com/component: o11y o11y.tidbcloud.com/cluster: "{cluster_id}" tidbcloud.com/nodegroup-name: tidbcloud-o11y-tidb-{cluster_id} machineType: {machine_type} metadata: disable-legacy-endpoints: "true" oauthScopes: - https://www.googleapis.com/auth/cloud-platform serviceAccount: {service_account} taints: - effect: NO_EXECUTE key: o11y-dedicated value: o11y - effect: NO_EXECUTE key: o11y-cluster value: "{cluster_id}" shieldedInstanceConfig: enableIntegrityMonitoring: true workloadMetadataConfig: mode: GKE_METADATA initialNodeCount: 1 locations: {locations_json} management: autoRepair: false maxPodsConstraint: maxPodsPerNode: 32 upgradeSettings: maxSurge: 1 version: {version} providerConfigRef: name: {provider_config_name} """ def gen_o11y_k8s_agent( region, tenant_id, project_id, k8s_cluster_name, gcp_project, network, subnets, provider_config_name, cluster, cluster_ref, service_account, zone, version, endpoint_name, image_type="cos_containerd", machine_type="n2-standard-2", disk_size_gb=100, min_size=1, max_size=3, desired_size=1, desired_spec="default", ): """Generate O11yK8sAgent YAML for GCP with complete nodepool annotations. These annotations are required by the NodePoolTask.reconcile logic in pkg/commontasks/gcp/nodepool.go """ subnets_json = json.dumps(subnets) return f""" apiVersion: agent.observability.pingcap.com/v2 kind: O11yK8sAgent metadata: annotations: # Basic o11y annotations o11y.cloud.pingcap.com/version: "0" # GCP NodePool configuration annotations (used by nodepool.go reconcile) o11y.cloud.pingcap.com/node-gke-cluster: "{cluster}" o11y.cloud.pingcap.com/node-gke-cluster-ref: "{cluster_ref}" o11y.cloud.pingcap.com/node-sa: "{service_account}" o11y.cloud.pingcap.com/node-provider-config-reference: "{provider_config_name}" o11y.cloud.pingcap.com/node-image-type: "{image_type}" o11y.cloud.pingcap.com/node-version: "{version}" o11y.cloud.pingcap.com/node-zone: "{zone}" # NodePool sizing annotations o11y.cloud.pingcap.com/node-min-size: "{min_size}" o11y.cloud.pingcap.com/node-max-size: "{max_size}" o11y.cloud.pingcap.com/node-desired-size: "{desired_size}" o11y.cloud.pingcap.com/node-disk-size: "{disk_size_gb}" # Spec configuration o11y.cloud.pingcap.com/max-spec: "medium" o11y.cloud.pingcap.com/min-spec: "default" o11y.cloud.pingcap.com/desired-spec: "{desired_spec}" # PSC endpoint name from existing o11yagent o11y.cloud.pingcap.com/o11y-service-psc-endpoint-name: "{endpoint_name}" name: "{project_id}" namespace: observability spec: gke: project: "{gcp_project}" network: "{network}" subnets: {subnets_json} kubernetesClusterName: "{k8s_cluster_name}" o11y: applicationId: k8s-infra bizType: tidbcloud projectId: "{project_id}" tenantId: "{tenant_id}" """ def gen_o11y_tidb_agent( region, tenant_id, project_id, cluster_id, k8s_cluster_name, gcp_project, network, subnets, provider_config_name, cluster, cluster_ref, service_account, zone, version, endpoint_name, image_type="cos_containerd", machine_type="n2-standard-2", disk_size_gb=100, min_size=1, max_size=3, desired_size=1, desired_spec="default", ): """Generate O11yTiDBAgent YAML for GCP with complete nodepool annotations. These annotations are required by the NodePoolTask.reconcile logic in pkg/commontasks/gcp/nodepool.go """ subnets_json = json.dumps(subnets) return f""" apiVersion: agent.observability.pingcap.com/v2 kind: O11yTiDBAgent metadata: annotations: # Basic o11y annotations o11y.cloud.pingcap.com/version: "0" # GCP NodePool configuration annotations (used by nodepool.go reconcile) o11y.cloud.pingcap.com/node-gke-cluster: "{cluster}" o11y.cloud.pingcap.com/node-gke-cluster-ref: "{cluster_ref}" o11y.cloud.pingcap.com/node-sa: "{service_account}" o11y.cloud.pingcap.com/node-provider-config-reference: "{provider_config_name}" o11y.cloud.pingcap.com/node-image-type: "{image_type}" o11y.cloud.pingcap.com/node-version: "{version}" o11y.cloud.pingcap.com/node-zone: "{zone}" # NodePool sizing annotations o11y.cloud.pingcap.com/node-min-size: "{min_size}" o11y.cloud.pingcap.com/node-max-size: "{max_size}" o11y.cloud.pingcap.com/node-desired-size: "{desired_size}" o11y.cloud.pingcap.com/node-disk-size: "{disk_size_gb}" # Spec configuration o11y.cloud.pingcap.com/max-spec: "large" o11y.cloud.pingcap.com/min-spec: "default" o11y.cloud.pingcap.com/desired-spec: "{desired_spec}" # PSC endpoint name from existing o11yagent o11y.cloud.pingcap.com/o11y-service-psc-endpoint-name: "{endpoint_name}" name: "{cluster_id}" namespace: observability spec: clusterTopology: pd: cpu: "0" memory: "0" replicas: 3 tidb: cpu: "0" memory: "0" replicas: 1 tiflash: cpu: "0" memory: "0" replicas: 0 tikv: cpu: "0" memory: "0" replicas: 3 version: unknown gke: project: "{gcp_project}" network: "{network}" subnets: {subnets_json} kubernetesClusterName: "{k8s_cluster_name}" o11y: applicationId: "{cluster_id}" bizType: tidbcloud projectId: "{project_id}" tenantId: "{tenant_id}" """ def make_backup_files(region, tenant_id): run(f"mkdir -p backup/{region}/{tenant_id}") result = run( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} get o11yroute -l tags.tidbcloud.com/tenant={tenant_id} -o yaml" ) with open(f"backup/{region}/{tenant_id}/o11yroutes.yaml", "w") as f: f.write(result.stdout) result = run( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} get privateendpoint -o yaml" ) with open(f"backup/{region}/{tenant_id}/privateendpoints.yaml", "w") as f: f.write(result.stdout) result = run( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} get o11yagent -o yaml" ) with open(f"backup/{region}/{tenant_id}/o11yagents.yaml", "w") as f: f.write(result.stdout) result = run( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} get cluster -o yaml" ) with open(f"backup/{region}/{tenant_id}/clusters.yaml", "w") as f: f.write(result.stdout) def make_k8s_files(region, tenant_id, project_id): vpc = get_seed_vpc(region, project_id) appbox = get_seed_appbox(region, project_id) nodepool = get_seed_nodepool(region, project_id, vpc["metadata"]["name"]) private_endpoint = get_o11y_private_endpoint(region, tenant_id, project_id) # Extract GCP-specific information gcp_project = private_endpoint["spec"].get("gcp", {}).get("projectID", "") subnet = private_endpoint["spec"].get("gcp", {}).get("subnet", "") subnets = [subnet] # Get cluster info from existing nodepool cluster = nodepool["spec"]["forProvider"].get("cluster", "") cluster_ref = nodepool["spec"]["forProvider"].get("clusterRef", {}).get("name", "") service_account = ( nodepool["spec"]["forProvider"].get("config", {}).get("serviceAccount", "") ) locations = nodepool["spec"]["forProvider"].get("locations", [f"{region}-a"]) if len(locations) > 1: locations = [locations[0]] version = nodepool["spec"]["forProvider"].get("version", "") provider_config_name = nodepool["spec"]["providerConfigRef"]["name"] appbox = appbox["metadata"]["name"] network = vpc["metadata"]["name"] # Generate NodePool YAML k8s_nodepool_yaml = gen_gke_k8s_nodepool( region=region, tenant_id=tenant_id, project_id=project_id, appbox=appbox, network=network, cluster=cluster, cluster_ref=cluster_ref, service_account=service_account, locations=locations, version=version, provider_config_name=provider_config_name, ) with open(f"k8s-nodepool-{project_id}.yaml", "w") as f: f.write(k8s_nodepool_yaml) # Get existing o11yagent for k8s-infra to extract endpointName o11yagent = get_o11y_o11yagent(region, tenant_id, project_id, "k8s-infra") endpoint_name = "" if o11yagent: endpoint_name = o11yagent.get("spec", {}).get("endpointName", "") # Generate O11yK8sAgent YAML o11y_k8s_agent_yaml = gen_o11y_k8s_agent( region=region, tenant_id=tenant_id, project_id=project_id, k8s_cluster_name=cluster_ref, gcp_project=gcp_project, network=network, subnets=subnets, provider_config_name=provider_config_name, cluster=cluster, cluster_ref=cluster_ref, service_account=service_account, zone=locations[0] if locations else f"{region}-a", version=version, endpoint_name=endpoint_name, ) with open(f"k8s-agent-{project_id}.yaml", "w") as f: f.write(o11y_k8s_agent_yaml) def make_tidb_files(region, tenant_id, project_id, cluster_id): vpc = get_seed_vpc(region, project_id) appbox = get_seed_appbox(region, project_id) nodepool = get_seed_nodepool(region, project_id, vpc["metadata"]["name"]) private_endpoint = get_o11y_private_endpoint(region, tenant_id, project_id) # Extract GCP-specific information gcp_project = private_endpoint["spec"].get("gcp", {}).get("projectID", "") subnet = private_endpoint["spec"].get("gcp", {}).get("subnet", "") subnets = [subnet] # Get cluster info from existing nodepool cluster = nodepool["spec"]["forProvider"].get("cluster", "") cluster_ref = nodepool["spec"]["forProvider"].get("clusterRef", {}).get("name", "") service_account = ( nodepool["spec"]["forProvider"].get("config", {}).get("serviceAccount", "") ) locations = nodepool["spec"]["forProvider"].get("locations", [f"{region}-a"]) if len(locations) > 1: locations = [locations[0]] version = nodepool["spec"]["forProvider"].get("version", "") provider_config_name = nodepool["spec"]["providerConfigRef"]["name"] appbox = appbox["metadata"]["name"] network = vpc["metadata"]["name"] # Generate TiDB NodePool YAML tidb_nodepool_yaml = gen_gke_tidb_nodepool( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, appbox=appbox, network=network, cluster=cluster, cluster_ref=cluster_ref, service_account=service_account, locations=locations, version=version, provider_config_name=provider_config_name, ) with open(f"tidb-nodepool-{cluster_id}.yaml", "w") as f: f.write(tidb_nodepool_yaml) # Get existing o11yagent for this cluster to extract endpointName o11yagent = get_o11y_o11yagent(region, tenant_id, project_id, cluster_id) endpoint_name = "" if o11yagent: endpoint_name = o11yagent.get("spec", {}).get("endpointName", "") # Generate O11yTiDBAgent YAML o11y_tidb_agent_yaml = gen_o11y_tidb_agent( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, k8s_cluster_name=cluster_ref, gcp_project=gcp_project, network=network, subnets=subnets, provider_config_name=provider_config_name, cluster=cluster, cluster_ref=cluster_ref, service_account=service_account, zone=locations[0] if locations else f"{region}-a", version=version, endpoint_name=endpoint_name, ) with open(f"tidb-agent-{cluster_id}.yaml", "w") as f: f.write(o11y_tidb_agent_yaml) def gen_k8s_apply_nodepool(region, project_id): print( f"kubectl --context {seed_ctx(region)} -n observability apply -f k8s-nodepool-{project_id}.yaml" ) def gen_tidb_apply_nodepool(region, cluster_id): print( f"kubectl --context {seed_ctx(region)} -n observability apply -f tidb-nodepool-{cluster_id}.yaml" ) def gen_k8s_patch_o11y_cluster(region, tenant_id, project_id): k8s_cluster = get_o11y_cluster(region, tenant_id, project_id, "k8s-infra") print( f"""kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} patch cluster {k8s_cluster["metadata"]["name"]} --type='json' -p='[{{"op": "remove", "path": "/spec/agent"}}]'""" ) def gen_tidb_patch_o11y_cluster(region, tenant_id, project_id, cluster_id): tidb_cluster = get_o11y_cluster(region, tenant_id, project_id, cluster_id) print( f"""kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} patch cluster {tidb_cluster["metadata"]["name"]} --type='json' -p='[{{"op": "remove", "path": "/spec/agent"}}]'""" ) def gen_k8s_pause_o11yroute(region, project_id): appbox = get_seed_appbox(region, project_id) o11yroute_name = appbox["metadata"]["name"] print( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} annotate o11yroute {o11yroute_name} dedicated.regionalserver.tidbcloud.com/paused=true --overwrite" ) print( f"""kubectl --context {seed_ctx(region)} -n {seed_ns(region)} patch o11yroute {o11yroute_name} --type='json' -p='[{{"op": "remove", "path": "/metadata/finalizers"}}]'""" ) def gen_tidb_pause_o11yroute(region, cluster_id): print( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} annotate o11yroute {cluster_id} dedicated.regionalserver.tidbcloud.com/paused=true --overwrite" ) print( f"""kubectl --context {seed_ctx(region)} -n {seed_ns(region)} patch o11yroute {cluster_id} --type='json' -p='[{{"op": "remove", "path": "/metadata/finalizers"}}]'""" ) def gen_check_vmagent_buffer(cluster_id): pod = get_dataplane_vmagent_pod(cluster_id) pod_name = pod["metadata"]["name"] print( f"kubectl --context dedicated.{cluster_id} -n tidb{cluster_id} exec -it {pod_name} -- /bin/sh -c 'cat /tmpData/persistent-queue/*/metainfo.json'" ) def gen_k8s_migrate(region, tenant_id, project_id): agent = get_o11y_o11yagent(region, tenant_id, project_id, "k8s-infra") print( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} delete o11yagent {agent['metadata']['name']}" ) print( f"kubectl --context {seed_ctx(region)} -n observability apply -f k8s-agent-{project_id}.yaml" ) def gen_tidb_migrate(region, tenant_id, project_id, cluster_id): agent = get_o11y_o11yagent(region, tenant_id, project_id, cluster_id) print( f"kubectl --context {o11y_ctx(region)} -n {o11y_ns(tenant_id)} delete o11yagent {agent['metadata']['name']}" ) print( f"kubectl --context {seed_ctx(region)} -n observability apply -f tidb-agent-{cluster_id}.yaml" ) def gen_k8s_delete_o11yroute(region, project_id): appbox = get_seed_appbox(region, project_id) o11yroute_name = appbox["metadata"]["name"] print( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} delete o11yroute {o11yroute_name}" ) def gen_tidb_delete_o11yroute(region, cluster_id): print( f"kubectl --context {seed_ctx(region)} -n {seed_ns(region)} delete o11yroute {cluster_id}" ) def do_tenant(region, tenant_id): make_backup_files(region=region, tenant_id=tenant_id) def do_project(region, tenant_id, project_id): o11yagent = get_o11y_o11yagent( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id="k8s-infra", ) if not o11yagent: return make_k8s_files( region=region, tenant_id=tenant_id, project_id=project_id, ) print("# apply nodepool") gen_k8s_apply_nodepool( region=region, project_id=project_id, ) print() print("# patch o11y cluster") gen_k8s_patch_o11y_cluster( region=region, tenant_id=tenant_id, project_id=project_id, ) print() print("# pause o11yroute") gen_k8s_pause_o11yroute( region=region, project_id=project_id, ) print() print("# migrate k8s") gen_k8s_migrate( region=region, tenant_id=tenant_id, project_id=project_id, ) print() print("# delete o11yroute") gen_k8s_delete_o11yroute( region=region, project_id=project_id, ) print() print(f"""# Wait for o11yk8sagent to become Normal echo "Waiting for o11yk8sagent {project_id} to become Normal..." while true; do PHASE=$(kubectl --context controlplane.seed-gcp-{region} -n observability get o11yk8sagent {project_id} -o jsonpath='{{.status.phase}}' 2>/dev/null) if [ "$PHASE" = "Normal" ]; then echo "o11yk8sagent {project_id} is now Normal" break fi echo "Current phase: $PHASE, waiting 10 seconds..." sleep 10 done """) def do_cluster(region, tenant_id, project_id, cluster_id): o11yagent = get_o11y_o11yagent( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, ) if not o11yagent: return make_tidb_files( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id ) print("# apply nodepool") gen_tidb_apply_nodepool( region=region, cluster_id=cluster_id, ) print() print("# patch o11y cluster") gen_tidb_patch_o11y_cluster( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, ) print() print("# pause o11yroute") gen_tidb_pause_o11yroute( region=region, cluster_id=cluster_id, ) print() print("# migrate tidb") gen_tidb_migrate( region=region, tenant_id=tenant_id, project_id=project_id, cluster_id=cluster_id, ) print() print("# delete o11yroute") gen_tidb_delete_o11yroute( region=region, cluster_id=cluster_id, ) print() print(f"""# Wait for o11ytidbagent to become Normal echo "Waiting for o11ytidbagent {cluster_id} to become Normal..." while true; do PHASE=$(kubectl --context controlplane.seed-gcp-{region} -n observability get o11ytidbagent {cluster_id} -o jsonpath='{{.status.phase}}' 2>/dev/null) if [ "$PHASE" = "Normal" ]; then echo "o11ytidbagent {cluster_id} is now Normal" break fi echo "Current phase: $PHASE, waiting 10 seconds..." sleep 10 done """) # Example cluster list for GCP - add your clusters here clusters = [ { "tenant_id": "60205", "project_id": "3200225", "cluster_id": "10627273943023685684", }, ] def main(): region = "us-west1" # Change to your GCP region projects = {} for c in clusters: projects[c["project_id"]] = { "tenant_id": c["tenant_id"], "project_id": c["project_id"], } for p in projects.values(): print("###### Project:", p["project_id"]) do_project( region=region, tenant_id=p["tenant_id"], project_id=p["project_id"], ) print() print() print() for c in clusters: print("###### Cluster:", c["cluster_id"]) do_cluster( region=region, tenant_id=c["tenant_id"], project_id=c["project_id"], cluster_id=c["cluster_id"], ) print() print() print() if __name__ == "__main__": main()