import hashlib import json import logging import subprocess import dotmap from kubernetes import client, config from src import util logger = logging.getLogger("oncall_master.k8s_client") logger.setLevel(logging.DEBUG) # namespaced network name to k8s manager network2cli = {} GKENameLimit = 40 @util.run_once def get_infra_api_cli() -> client.ApiClient: # config.load_config(kube_config_path="~/.kube/config") config.load_kube_config() conf = client.Configuration.get_default_copy() # conf.client_side_validation = False conf.verify_ssl = False # enable debug mode for k8s client to logging more verbose log # conf.debug = True cli = client.ApiClient(configuration=conf) return cli def get_dataplane_cli(network: dotmap.DotMap) -> client.ApiClient: name = build_dataplane_cluster_name_by_network(network) if name not in network2cli: new_and_register_eks_client_manager(name, network) return network2cli[name] def new_and_register_eks_client_manager(dataplane_k8s_name: str, network: dotmap.DotMap) -> client.ApiClient: if network.spec.provider == "eks": conf = get_eks_kube_config(dataplane_k8s_name) cli = client.ApiClient(configuration=conf) network2cli[dataplane_k8s_name] = cli return cli def get_eks_kube_config(dataplane_cluster_name: str) -> client.Configuration: AnnotationKeyEKSEndpoint = "infra.pingcap.com/eks-endpoint" local_cli = get_infra_api_cli() cr_api = client.CustomObjectsApi(api_client=local_cli) conf = client.Configuration() eks = util.get_cluster_cr(cr_api, "eks.aws.crossplane.io", "v1beta1", "clusters", dataplane_cluster_name) endpointPublicAccess = eks.spec.forProvider.resourcesVpcConfig.endpointPublicAccess # If EKS enable public access, use the public kubeconfig. if eks.metadata.get(AnnotationKeyEKSEndpoint, "") != "" and not endpointPublicAccess: conf.host = eks.metadata.annotations[AnnotationKeyEKSEndpoint] else: conf.host = eks.status.atProvider.endpoint # TODO: Make it secure once we can get the CA data of the cluster. conf.verify_ssl = False params = { 'region': eks.spec.forProvider.region, 'cluster_name': eks.metadata.name, } providerConfigName = eks.spec.get("providerConfigRef", {}).get("name", "") if providerConfigName != "": provider_config = util.get_cluster_cr(cr_api, "aws.crossplane.io", "v1beta1", "providerconfigs", providerConfigName) if provider_config.spec.assumeRoleARN != "": params["role"] = provider_config.spec.assumeRoleARN if provider_config.spec.externalID != "": params["externalId"] = provider_config.spec.externalID token = get_eks_token(params) conf.api_key["authorization"] = token conf.api_key_prefix['authorization'] = 'Bearer' return conf def get_eks_token(params: dict) -> str | None: """ :param params: region, cluster_name, role, external_id :return: """ try: cmdBuilder = ['aws-iam-authenticator', 'token', '--region', params['region'], '--cluster-id', params['cluster_name']] role = params.get('role', '') if role != "": cmdBuilder.extend(['--role', role]) external_id = params.get('externalId', '') if external_id != "": cmdBuilder.extend(['--external-id', external_id]) # execute the command via subprocess cmd = ' '.join(cmdBuilder) logger.debug(f"cmd to get eks token: {cmd}") result = subprocess.run(cmd, shell=True, check=True, stdout=subprocess.PIPE) # parse the response response = json.loads(result.stdout.decode('utf-8')) # get the token from response token = response['status']['token'] logger.debug(f'Token: {token}') except subprocess.CalledProcessError as e: logger.error(f'Error occurred: {str(e)}') return None logger.debug(f"eks token: {token}") return token def build_dataplane_cluster_name_by_network(network: dotmap.DotMap): if network.spec.provider == "gke": return unique_identifier_v2(network.metadata, with_len_limit(GKENameLimit)) else: return unique_identifier_v2(network.metadata) hash_size = 8 # transform from https://github.com/tidbcloud/infra-provider/blob/main/pkg/namer/namer.go#L429-L429 def unique_identifier_v2(metadata: dotmap.DotMap, *options): label_length_limit = 63 result_limit = label_length_limit - hash_size - 1 namespaced_name = f"{metadata.namespace}/{metadata.name}" hash = hashlib.md5(namespaced_name.encode()).hexdigest()[:hash_size] if metadata.namespace: result = f"{metadata.namespace}-{metadata.name}" else: result = metadata.get_name() for opt in options: result = opt(result) if len(result) > result_limit: result = result[:result_limit] return f"{result}-{hash}" # transform from https://github.com/tidbcloud/infra-provider/blob/main/pkg/namer/namer.go#L316 def with_len_limit(limit: int): def wrapper(name: str) -> str: result_imit = limit - hash_size - 1 if len(name) > result_imit: return name[:result_imit] else: return name return wrapper