import copy import datetime import hashlib import json import os import re import pandas as pd from dateutil import parser from dateutil.tz import tzutc from analyst.libs import util, common_cloud_resource_analyzer, metadata from analyst.libs.common_cloud_resource_analyzer import parse_env_region_by_3layer_k8s_name from analyst.libs.consts import VPCType, Component, UsedBy, GCPProduct, ServiceType from analyst.resource_leak.settings import GCP_ASSET_SNAPSHOT_ANALYSIS_RESULT_FILE, ASSET_FILE_DIR GCP_COMPUTE_INSTANCES_FILE = os.path.join(ASSET_FILE_DIR, 'gcp_compute_instances.pkl') GCP_COMPUTE_DISKS_FILE = os.path.join(ASSET_FILE_DIR, 'gcp_compute_disks.pkl') GCP_COMPUTE_LBS_FILE = os.path.join(ASSET_FILE_DIR, 'gcp_compute_forwarding_rules.pkl') GCP_COMPUTE_NATS_FILE = os.path.join(ASSET_FILE_DIR, 'gcp_compute_routers.pkl') GCP_COMPUTE_GKES_FILE = os.path.join(ASSET_FILE_DIR, 'gcp_container_clusters.pkl') GCP_COMPUTE_NETWORKS_FILE = os.path.join(ASSET_FILE_DIR, 'gcp_compute_networks.pkl') # StakeHolder_type: ['tenant','cluster'] # K8S_type: ['base','seed','shoot'] # ebs usage: ['etcd','monitoring'] result_header = { # time 'datetime': 0, 'created_at': 0, 'hold_time': 0, # aws/gcp cloud resource control handle 'billing_key': 'null', 'region': 'null', 'account_id': 'null', 'id': 'null', 'name': 'null', # resource dashboard 'service_type': 'null', 'usedby': 'null', 'network_status': 'null', 'vpc_name': 'null', 'k8s': 'null', # because it's always be recorded in tag 'network_namespace': 'null', 'network_name': 'null', 'cluster_status': 'null', 'product': 'null', 'component': 'null', # tidbcloud label standard 'env': 'null', 'tenant_id': 'null', 'project_id': 'null', 'cluster_id': '', 'member': '', # 'component': 'null', # 'usedby': 'null', # other resource attribute 'vpc_type': 'null', 'shoot_id': 'null', # for eks/gke network, shoot_id isn't important 'vpc_id': 'null', 'ebs_size': 0, 'ec2_instance_type': 'null', 'ec2_instance_state': 'null', 'ec2_wg_id': 'null', 'ebs_pvc_name': "null", 'ebs_state': 'null', 'ebs_real_k8s': 'null', 'ebs_ec2_mount': 'null', 'elb_service_name': 'null', # used by gcp client 'labelFingerprint': 'null', 'labels': 'null', 'extra_info': {}, } label_mapping = { 'environment': 'env', 'tenant': 'tenant_id', 'cluster': 'cluster_id', 'project': 'project_id', 'usedby': 'usedby', 'component': 'component', 'servicetype': 'service_type' } result_header['datetime'] = int( datetime.datetime.now().replace(tzinfo=tzutc()).strftime('%s')) # Analysis class DatetimeEncoder(json.JSONEncoder): def default(self, obj): try: return super().default(obj) except TypeError: return str(obj) # zone: https://www.googleapis.com/compute/v1/projects/dbaas-prod/zones/asia-northeast1-a GCP_REGION_ZONE_RE = re.compile(r'.*/((\w+-\w+\d)-\w)$') GCP_REGION_IN_SUBNETWORK_NAME_RE = re.compile(r'.*/regions/(.*)/subnetworks/.*') def parse_gcp_zone(zone: str) -> (str, str): """ :param zone: :return: - region str: - zone str """ result = GCP_REGION_ZONE_RE.match(zone) if result: return result.groups()[1], result.groups()[0] return 'unknown', 'unknown' def parse_gcp_region(region: str) -> str: """ :param region: https://www.googleapis.com/compute/v1/projects/pingcap-gardener/regions/us-west1 :return: - region str: - zone str """ if region: splited = region.split('/') if splited: return splited[-1] return '' # https://www.googleapis.com/compute/v1/projects/tidbcloud-dev-000/regions/us-west2/subnetworks/psc-nat def parse_gcp_region_from_subnetwork_name(subnetwork_name: str) -> str: result = GCP_REGION_IN_SUBNETWORK_NAME_RE.match(subnetwork_name) if result: return result.groups()[0] return '' # vpc name is GCP network name def parse_vpc_name(network_interfaces: str) -> str: interfaces = json.loads(network_interfaces) # todo(chenfei): how to deal if there is more than one interface for interface in interfaces: # https://www.googleapis.com/compute/v1/projects/pingcap-gardener/global/networks/nightly-ms-3199044-gke-us-west1-93ef30cb network = interface['network'] splited = network.split('/') if splited: return splited[-1] return '' def extract_information_in_vm(md: metadata.Metadata): vm_results = [] for _, instance in pd.read_pickle(GCP_COMPUTE_INSTANCES_FILE, compression="gzip").iterrows(): result = copy.deepcopy(result_header) region, zone = parse_gcp_zone(instance['zone']) vpc_name = parse_vpc_name(instance['network_interfaces']) result.update({ 'product': 'vm', 'name': instance['name'], 'id': str(instance['id']), 'vpc_name': vpc_name, 'vpc_id': vpc_name, 'ec2_instance_type': instance['machine_type'].split('machineTypes/')[-1], 'labelFingerprint': instance['label_fingerprint'], 'ec2_instance_state': instance['status'], # zone is ok 'region': region, 'zone': zone, 'labels': instance['labels'], 'account_id': instance['project_id'], 'created_at': parser.parse(instance['creation_timestamp']).timestamp() }) # get existed label: instance_labels = json.loads(instance['labels']) for key, value in label_mapping.items(): if key in instance_labels.keys(): result[value] = instance_labels[key] if result['component'] == 'extra-tidb' and result['usedby'] != 'dbaas-control-plane-shoot': result['usedby'] = 'dbaas-control-plane-shoot' seed_match_result = re.findall( "gke-(.*)-seed-(.*)", result['name']) if len(seed_match_result) > 0: seed_result = list(seed_match_result[0]) result.update({ 'k8s_type': 'seed', 'cluster_id': 'admin', 'tenant_id': 'admin', 'project_id': 'admin', 'env': seed_result[0], 'usage': 'dbaas-control-plane-seed', 'usedby': 'dbaas-control-plane-seed', 'component': 'dbaas-control-plane-seed', 'service_type': 'dedicated', }) # for seed cluster vm_metadata = json.loads(instance['metadata']) if 'items' in vm_metadata: for meta in vm_metadata['items']: if meta['key'] == 'cluster-name': result['k8s'] = meta['value'] # for shoot cluster shoot_match_result = re.findall( "shoot--(.*)--(.*)", result['name']) if len(shoot_match_result) > 0: shoot_result = list(shoot_match_result[0]) result.update({ 'k8s': 'shoot--%s--%s' % (shoot_result[0], shoot_result[1].split('-')[0]), 'k8s_type': 'shoot', 'env': shoot_result[0].replace('nightly', 'dev'), 'shoot_id': shoot_result[1].split('-')[0], 'service_type': 'dedicated', }) if shoot_result[1].split('-')[1] == 'master': result.update({ 'usedby': "dbaas-control-plane-shoot", 'usage': "dbaas-control-plane-shoot", 'component': 'dbaas-control-plane-shoot', 'cluster_id': 'admin', }) elif 'ngm' in shoot_result[1]: result.update({ 'usedby': "dbaas-control-plane-shoot", 'usage': "ngm", 'component': 'ngm', 'service_type': 'dedicated', }) if result['k8s_type'] == 'shoot' and 'gardener_wg_name' in instance_labels: gardenerWgName = instance_labels['gardener_wg_name'] gardenerWgNameSeps = gardenerWgName.split('-') for component_key in ['pd', 'tiflash', 'tikv', 'tidb', 'lightning', 'ticdc']: if component_key in gardenerWgNameSeps: component_key = component_key.replace( 'lightning', 'tidb-lightning') result.update({ 'component': component_key, 'usage': component_key, 'usedby': 'customer-tidb', 'cluster_id': gardenerWgNameSeps[0].replace('tidb', '') }) for component_key in ['ngm', 'extra']: if component_key in gardenerWgNameSeps: component_key = component_key.replace( 'extra', 'extra-tidb') result.update({ 'component': component_key, 'usage': component_key, 'usedby': 'dbaas-control-plane-shoot', 'cluster_id': gardenerWgNameSeps[0].replace('tidb', '') }) common_cloud_resource_analyzer.analyze_by_pod_node_relation(result, instance['name'], md) vm_results.append(result) return vm_results INSTANCE_NAME_IN_DISK_USERS = re.compile(r'.*/instances/(.*)$') def get_vm_name_from_disk_users(users: str) -> str: """ :param users: 'https://www.googleapis.com/compute/v1/projects/dbaas-prod/zones/us-central1-a/instances/gke-prod-13728130891-tidbcloud-prod-1-47f41914-rs37' :return: """ if not users: return '' result = INSTANCE_NAME_IN_DISK_USERS.match(users) if result: return result.groups()[0] return '' def extract_information_in_disk(pvc_lists, vm_result, md: metadata.Metadata): name2vm = {} for each in vm_result: name2vm[each['name']] = each disk_results = [] for _, disk in pd.read_pickle(GCP_COMPUTE_DISKS_FILE, compression="gzip").iterrows(): result = copy.deepcopy(result_header) region, zone = parse_gcp_zone(disk['zone']) result.update({ 'product': 'disk', 'name': disk['name'], 'id': disk['name'], 'labelFingerprint': disk['label_fingerprint'], 'ebs_state': disk['status'], 'ebs_size': disk['size_gb'], 'region': region, 'zone': zone, 'labels': disk['labels'], 'account_id': disk['project_id'], 'created_at': parser.parse(disk['creation_timestamp']).timestamp() }) vm_name = '' if len(disk['users']) != 0: vm_name = get_vm_name_from_disk_users(disk['users'][0]) # there's no need to consider about multi-attach result.update({'ebs_ec2_mount': vm_name}) else: # no user uses this disk result['ebs_state'] = 'available' # get existed label: disk_labels = json.loads(disk['labels']) for key, value in label_mapping.items(): if key in disk_labels.keys(): result[value] = disk_labels[key] if disk_labels.get('goog-k8s-cluster-name'): result['k8s'] = disk_labels['goog-k8s-cluster-name'] if result['component'] == 'extra-tidb' and result['usedby'] != 'dbaas-control-plane-shoot': result['usedby'] = 'dbaas-control-plane-shoot' # seed results seed_match_result = re.findall( "gke-(.*)-seed-(.*)", result['name']) if len(seed_match_result) > 0: seed_result = list(seed_match_result[0]) result.update({ 'k8s_type': 'seed', 'cluster_id': 'admin', 'tenant_id': 'admin', 'project_id': 'admin', 'env': seed_result[0], 'usage': 'dbaas-control-plane-seed', 'usedby': 'dbaas-control-plane-seed', 'component': 'dbaas-control-plane-seed', 'service_type': 'dedicated', }) # parse from description # for example: # {"kubernetes.io/created-for/pv/name":"pvc-60f4a589-c7e0-46e8-b928-4d626d59dee4", # "kubernetes.io/created-for/pvc/name":"tiflow-nightly-ms-3199044-gke-us-west1-93ef30cb-vmage-victoria", # "kubernetes.io/created-for/pvc/namespace":"tiflow-nightly-ms-3199044-gke-us-west1-93ef30cb", # "storage.gke.io/created-by":"pd.csi.storage.gke.io"} if 'description' in disk: try: disk_description = json.loads(disk['description']) except (ValueError, TypeError) as e: disk_description = {} if 'kubernetes.io/created-for/pv/name' in disk_description: pvc_name = disk_description['kubernetes.io/created-for/pvc/name'] result['ebs_pvc_name'] = pvc_name namespace = disk_description['kubernetes.io/created-for/pvc/namespace'] result['ebs_pvc_namespace'] = namespace shoot_info = namespace.split('--') if len(shoot_info) > 0 and shoot_info[0] == 'shoot': result['env'] = shoot_info[1].replace('nightly', 'dev') result['shoot_id'] = shoot_info[2] result['k8s_type'] = 'shoot' # shoot ebs pvc_match_result = re.findall( "shoot--(.*)--(.*)--(.*)", result['name']) if len(pvc_match_result) > 0: pvc_result = list(pvc_match_result[0]) result['k8s'] = 'shoot--%s--%s' % (pvc_result[0], pvc_result[1]) result['env'] = pvc_result[0] if result['shoot_id'] == 'null': result['shoot_id'] = pvc_result[1] volumename = pvc_result[-1] if volumename in pvc_lists.keys(): result['ebs_pvc_name'] = pvc_lists[volumename]['pvc_name'] pvc_namespace = pvc_lists[volumename]['namespace'] if 'tidb' in pvc_namespace and pvc_namespace != 'tidb-admin': result['cluster_id'] = pvc_namespace.replace( 'tidb', '') else: print('pvc not found, volumeid: %s' % volumename) # try get info from description else: # for shoot master ebs shoot_match_result = re.findall( "shoot--(.*)--(.*)-(.*)", result['name']) if len(shoot_match_result) > 0: shoot_result = list(shoot_match_result[0]) if result['k8s'] == 'null': result.update({ 'k8s': 'shoot--%s--%s' % (shoot_result[0], shoot_result[1].split('-')[0]), 'k8s_type': 'shoot', 'env': shoot_result[0].replace('nightly', 'dev'), 'shoot_id': shoot_result[1].split('-')[0], 'service_type': 'dedicated', }) else: result.update({ 'env': shoot_result[0].replace('nightly', 'dev'), 'service_type': 'dedicated', }) if shoot_result[1].split('-')[1] == 'master': result.update({ 'usedby': "dbaas-control-plane-shoot", 'usage': "dbaas-control-plane-shoot", 'component': 'dbaas-control-plane-shoot', 'cluster_id': 'admin', }) elif 'ngm' in shoot_result[1]: result.update({ 'usedby': "dbaas-control-plane-shoot", 'usage': "ngm", 'component': 'ngm' }) if result['vpc_type'] == VPCType.Shoot and 'labels' in disk and 'gardener_wg_name' in disk_labels: gardenerWgName = disk_labels['gardener_wg_name'] gardenerWgNameSeps = gardenerWgName.split('-') for component_key in ['pd', 'tiflash', 'tikv', 'tidb', 'lightning', 'ticdc']: if component_key in gardenerWgNameSeps: component_key = component_key.replace( 'lightning', 'tidb-lightning') result.update({ 'component': component_key, 'usage': component_key, 'usedby': 'customer-tidb', 'cluster_id': gardenerWgNameSeps[0].replace('tidb', '') }) for component_key in ['ngm', 'extra']: if component_key in gardenerWgNameSeps: component_key = component_key.replace( 'extra', 'extra-tidb') result.update({ 'component': component_key, 'usage': component_key, 'usedby': 'dbaas-control-plane-shoot', 'cluster_id': gardenerWgNameSeps[0].replace('tidb', '') }) # sync info from PVC util.set_member_name_from_pvc_name(result, result.get('ebs_pvc_name', '')) util.set_component_from_pvc_name(result, result.get('ebs_pvc_name', '')) util.set_cluster_id_from_tc_namespace(result, result.get('ebs_pvc_namespace', '')) # sync info from node node = name2vm.get(vm_name) if node: common_cloud_resource_analyzer.sync_from_node_to_ebs(result, node) disk_results.append(result) return disk_results def extract_information_in_lb(): # https://www.googleapis.com/compute/v1/projects/dbaas-prod/global/networks/shoot--prod--23042a5e def parse_vpc_name_from_network(network: str) -> str: splited = network.split('/') if splited: return splited[-1] lb_results = [] for _, lb in pd.read_pickle(GCP_COMPUTE_LBS_FILE, compression="gzip").iterrows(): result = copy.deepcopy(result_header) region = parse_gcp_region(lb['region']) result.update({ 'product': 'lb', 'name': lb['name'], 'id': str(lb['id']), 'labelFingerprint': lb['label_fingerprint'], 'region': region, 'zone': '', 'labels': lb['labels'], 'account_id': lb['project_id'], 'created_at': parser.parse(lb['creation_timestamp']).timestamp(), 'k8s_service_name': '', 'psc_connection_id': str(lb.get('psc_connection_id', '')), 'ip_address': lb.get('ip_address', ''), 'lb_scheme': lb.get('load_balancing_scheme', ''), 'port_range': lb.get('port_range', ''), }) # parse from description # {"kubernetes.io/service-name":"tidb1379661944646413400/db-tidb-private"} if 'description' in lb: try: lb_description = json.loads(lb['description']) except (ValueError, TypeError) as e: lb_description = {} service = lb_description.get("kubernetes.io/service-name", '') if service: namespace, svc_name = service.split('/') util.set_cluster_id_from_tc_namespace(result, namespace) result['k8s_service_name'] = svc_name if svc_name.startswith('db-tidb'): result['component'] = Component.Tidb # parse from network if lb['network']: vpc_name = parse_vpc_name_from_network(lb['network']) result['vpc_name'] = vpc_name # parse from name if result['name'].startswith('o11y'): util.set_if_empty(result, 'usedby', UsedBy.DBaaSControlPlaneShoot) util.set_if_empty(result, 'cluster_id', 'admin') lb_results.append(result) return lb_results def extract_information_in_nat(): # https://www.googleapis.com/compute/v1/projects/dbaas-prod/global/networks/shoot--prod--23042a5e def parse_vpc_name_from_network(network: str) -> str: splited = network.split('/') if splited: return splited[-1] nat_results = [] for _, nat in pd.read_pickle(GCP_COMPUTE_NATS_FILE, compression="gzip").iterrows(): result = copy.deepcopy(result_header) region = parse_gcp_region(nat['region']) result.update({ 'product': 'nat', 'name': nat['name'], 'id': str(nat['id']), 'labelFingerprint': '', # no this filed 'labels': '{}', # no this field 'region': region, 'zone': '', 'account_id': nat['project_id'], 'created_at': parser.parse(nat['creation_timestamp']).timestamp(), }) # parse from network if nat['network']: vpc_name = parse_vpc_name_from_network(nat['network']) result['vpc_name'] = vpc_name nat_results.append(result) return nat_results def extract_information_in_gke(): # https://www.googleapis.com/compute/v1/projects/dbaas-prod/global/networks/shoot--prod--23042a5e def parse_vpc_name_from_network(network: str) -> str: splited = network.split('/') if splited: return splited[-1] gke_results = [] for _, gke in pd.read_pickle(GCP_COMPUTE_GKES_FILE, compression="gzip").iterrows(): result = copy.deepcopy(result_header) result.update({ 'product': 'gke', 'name': gke['name'], 'k8s': gke['name'], 'id': str(gke['id']), 'labelFingerprint': gke['label_fingerprint'], # no this filed 'labels': gke['resource_labels'], # no this field 'region': gke['zone'], 'zone': '', 'account_id': gke['project_id'], # 2023-12-26T02:10:17+00:00 'created_at': parser.parse(gke['create_time']).timestamp(), 'vpc_name': gke['network'], 'cidr_block': gke['cluster_ipv4_cidr'], 'default_max_pods_constraint': gke['default_max_pods_constraint'], 'endpoint': gke['endpoint'], }) # {"component":"seed","env":"prod","tenant":"admin","tenant-component":"admin-seed"} # {"project":"3199891","servicetype":"dedicated","tenant":"60205","usedby":"dbaas-control-plane-shoot"} if result['labels']: try: labels = json.loads(result['labels']) except TypeError: labels = {} if labels.get('component') == 'seed': util.set_if_empty(result, 'service_type', ServiceType.DEDICATED) util.set_if_empty(result, 'usedby', UsedBy.DBaaSControlPlaneSeed) elif labels.get('usedby') == UsedBy.DBaaSControlPlaneShoot: util.set_if_empty(result, 'service_type', ServiceType.DEDICATED) util.set_if_empty(result, 'usedby', UsedBy.DBaaSControlPlaneShoot) util.set_if_empty(result, 'tenant_id', labels.get('tenant', '')) util.set_if_empty(result, 'project_id', labels.get('project', '')) gke_results.append(result) return gke_results def extract_information_in_vpc(): vpc_results = [] for _, vpc in pd.read_pickle(GCP_COMPUTE_NETWORKS_FILE, compression="gzip").iterrows(): region = '' if len(vpc['subnetworks']) != 0: region = parse_gcp_region_from_subnetwork_name(vpc['subnetworks'][0]) result = copy.deepcopy(result_header) result.update({ 'product': 'vpc', 'name': vpc['name'], 'id': str(vpc['id']), 'region': region, 'account_id': vpc['project_id'], 'created_at': parser.parse(vpc['creation_timestamp']).timestamp(), 'labels': '{}', # no labels set on GCP network(VPC) 'vpc_name': vpc['name'], 'vpc_id': str(vpc['id']), 'cidr_block': '', }) # try to parse info from description # an example of valid description: # 'Tags: map[project:2353477 servicetype:dedicated tenant:30060 usedby:dbaas-control-plane-shoot]' description = {} if 'description' in vpc: try: description_str = vpc['description'] if description_str: striped = description_str.removeprefix('Tags: ').removeprefix('map[').removesuffix(']').strip() split = striped.split(' ') for each in split: k, v = each.split(':') description[k] = v except (ValueError, TypeError) as e: description = {} util.set_if_empty(result, 'tenant_id', description.get('tenant', '')) util.set_if_empty(result, 'project_id', description.get('project', '')) util.set_if_empty(result, 'service_type', description.get('servicetype', '')) util.set_if_empty(result, 'usedby', description.get('usedby', '')) vpc_results.append(result) return vpc_results def extract_information_for_common(resource, network_list, md: metadata.Metadata): if 'shoot' in resource['k8s']: resource.update( { 'env': resource['k8s'].split('--')[1].replace('nightly', 'dev'), 'shoot_id': resource['k8s'].split('--')[2], 'vpc_type': 'shoot', 'usedby': 'dbaas-control-plane-shoot', 'service_type': "dedicated", 'vpc_name': resource['k8s'] } ) for network in network_list: if resource['shoot_id'] == network['shoot_name']: resource.update( { 'tenant_id': network['tenant'], 'project_id': network['project'], 'network_status': network['status'], 'network_name': network['network_name'], 'network_namespace': network['namespace'], } ) break # Serverless resources elif re.search(r'^(prod|staging|dev)-.*-f0\d+$', resource.get('k8s', '')): resource.update({ 'service_type': 'serverless', 'usedby': 'serverless', }) # 3-layer structure elif parse_env_region_by_3layer_k8s_name(resource['k8s']) != None: resource['env'], resource['region'] = parse_env_region_by_3layer_k8s_name(resource['k8s']) resource.update({ 'usedby': "dbaas-control-plane-shoot", 'service_type': "dedicated", }) # gcp has a special case: 'cluster_id', 'shared' if resource['cluster_id'] not in ['null', '', 'admin']: resource['usedby'] = "customer-tidb" if resource['project_id'] not in ['null', '']: resource['network_name'] = 'gke-' + resource['region'] resource['namespace'] = resource['env'] + '-' + resource['project_id'] #print(resource['namespace'], " ", resource['network_name']) for network in network_list: if (network['network_name'] == resource['network_name'] and network['namespace'] == resource['namespace']): resource.update( { 'tenant_id': network['tenant'], 'network_status': network['status'], } ) break if resource['cluster_id'] != 'null' and resource['cluster_id'] != 'admin': for network in network_list: if resource['cluster_id'] in network['cluster_list'].keys(): resource.update( { 'tenant_id': network['tenant'], 'project_id': network['project'], 'network_status': network['status'], 'network_name': network['network_name'], 'network_namespace': network['namespace'], 'cluster_status': network['cluster_list'][resource['cluster_id']], 'usedby': "customer-tidb", 'service_type': "dedicated", } ) if resource['vpc_name'] == 'null' and 'dedicated' not in resource['k8s']: vpc_name_hash_id = hashlib.md5( ("%s/%s" % (network['namespace'], network['network_name'])).encode('utf-8')).hexdigest()[:8] vpc_name = "%s-%s-%s" % (network['namespace'], network['network_name'], vpc_name_hash_id) resource.update( { 'vpc_id': vpc_name, 'vpc_name': vpc_name, } ) break common_cloud_resource_analyzer.set_env_by_account(resource) # kube-apiserver if 'kube-apiserver' in resource['elb_service_name']: resource.update( { 'env': resource['elb_service_name'].split('--')[1], 'shoot_id': resource['elb_service_name'].split('--')[2].replace('/kube-apiserver', ''), 'vpc_type': 'shoot', 'usedby': 'dbaas-control-plane-shoot', 'cluster_id': 'admin', 'component': 'kube-apiserver', 'service_type': "dedicated", 'k8s': resource['elb_service_name'].split('/')[0] } ) if resource['component'] == 'Gardener-ShootMaster': resource['component'] = 'dbaas-control-plane-shoot' # tidbmonitor and slowlog should be after tidb, so that these labels can overwrite the wrong label for key in ['pd', 'tikv', 'tidb', 'tiflash', 'tidbmonitor', 'slowlog']: if key in resource['ebs_pvc_name']: resource['component'] = key if resource['cluster_id'] == 'null': resource['cluster_id'] = 'admin' # if 'null' in result['k8s']: # result['component'] = 'unmanaged' if 'tidb' in resource['elb_service_name'] and 'tidb-admin' not in resource['elb_service_name']: resource['cluster_id'] = resource['elb_service_name'].split( '/')[0].replace('tidb', '') resource['usedby'] = 'customer-tidb' if resource['name'] == 'null' and resource['id'] != 'null': resource['name'] = resource['id'] if resource['service_type'] == 'serverless': resource['usedby'] = "serverless" resource['env'] = resource['env'].replace('nightly', 'dev') common_cloud_resource_analyzer.update_by_keyword(resource) util.set_component_from_member_name(resource, resource['member']) common_cloud_resource_analyzer.sync_from_network_by_k8s_name(resource, resource['k8s'], md) common_cloud_resource_analyzer.sync_from_network_by_vpc_name(resource, resource['vpc_name'], md) common_cloud_resource_analyzer.set_by_vpc_name(resource, md) if resource['vpc_type'] == 'shoot' and resource['component'] in ['pd', 'tidb', 'tikv', 'tiflash', 'ticdc', 'tidb-lightning']: resource['usedby'] = 'customer-tidb' if resource['vpc_type'] == VPCType.Shoot and resource['product'] in {GCPProduct.NAT, }: util.set_if_empty(resource, 'usedby', UsedBy.DBaaSControlPlaneShoot) util.set_if_empty(resource, 'component', Component.Admin) util.set_if_empty(resource, 'cluster_id', Component.Admin) if resource['vpc_type'] == VPCType.Base and resource['component'] == 'null': resource['component'] = 'dbaas-control-plane-base' if resource['cluster_id'] == 'admin': resource['cluster_status'] = resource['network_status'] util.set_if_empty(resource, 'component', 'admin') if resource['cluster_status'] == 'Normal': resource['cluster_status'] = 'Running' def analysis_resources_list(pvc_lists, network_list, md: metadata.Metadata): results = [] vm_result = extract_information_in_vm(md) results += vm_result results += extract_information_in_disk(pvc_lists, vm_result, md) results += extract_information_in_lb() results += extract_information_in_nat() results += extract_information_in_gke() results += extract_information_in_vpc() for resource in results: extract_information_for_common(resource, network_list, md) return results def main(): md = metadata.build_metadata() # create_table() # TODO: ignore it when the shoot doesn't exist if os.path.exists("new_network_result.json"): network_list = json.load( open('new_network_result.json', 'r')) else: network_list = {} if os.path.exists('kube_persistentvolumeclaim_info_asset_snapshot_analysis_result.json'): pvc_list = json.load( open('kube_persistentvolumeclaim_info_asset_snapshot_analysis_result.json', 'r')) else: pvc_list = {} results = analysis_resources_list(pvc_list, network_list, md) json.dump(results, fp=open( GCP_ASSET_SNAPSHOT_ANALYSIS_RESULT_FILE, 'w'), sort_keys=True, indent=4) if __name__ == "__main__": main()