import copy import json import os import pandas as pd from playhouse.shortcuts import model_to_dict from analyst.libs import consts, util from analyst.libs.consts import AWSProduct, GCPProduct from analyst.resource_leak import model from analyst.resource_leak.settings import ASSET_FILE_DIR CLUSTER_TENANT_INFO_PKL_FILE = os.path.join(ASSET_FILE_DIR, 'all_clusters_tenants_info.pkl') INFRA_EXPORTER_NETWORK_ASSET_SNAPSHOT_ANALYSIS_RESULT = os.path.join(ASSET_FILE_DIR, 'new_network_result.json') TENANT_RESOURCE_UNITS_ANALYSIS_JSON_FILE = os.path.join(ASSET_FILE_DIR, 'tenant_resource_units_analysis.json') network_template = { 'provider': '', 'vpc_name': '', 'kind': '', 'region': '', 'account_id': '', 'vpc_id': '', 'status': '', 'env': '', 'dbaas_info': {'tenant_id': '', 'tenant_name': '', 'project_id': '', 'project_name': '', 'tenant_kind': ''}, 'infra_resources': {'network': {}, 'namespace': {}, 'vpc_peering': [], 'migrated_network': {}}, 'clusters': {}, 'dataplane_resources': [], 'features': [] } networks = {} model.init_db(model.create_db_instance()) resources_in_db: list[model.CloudAsset] = model.CloudAsset.select().where( model.CloudAsset.service_type.in_([consts.ServiceType.DEDICATED]), model.CloudAsset.resource_deleted_at.is_null() ) results = [] # you can repeatly rescan the analysis result to get the more valuable conclusion. # build vpc-related-resources # resources = json.load(open('aws_asset_snapshot_analysis_result.json', 'r')) resources = [] for each in resources_in_db: data = model_to_dict(each) data['usedby'] = data['_usedby'] data['network_name'] = data['_network_name'] resources.append(data) def get_network_id(resource: dict) -> str: network_id = resource['vpc_id'] # for GCP resources, we use vpc_name as network_id if resource['provider'] == consts.Provider.GCP: network_id = resource['vpc_name'] return network_id for resource in resources: # build vpc struct if (resource['product'] in {consts.AWSProduct.VPC, consts.GCPProduct.VPC} and resource['vpc_type'] == consts.VPCType.Shoot): network_id = get_network_id(resource) networks[network_id] = copy.deepcopy(network_template) networks[network_id]['provider'] = resource['provider'] networks[network_id]['vpc_id'] = resource['rid'] networks[network_id]['vpc_name'] = resource['rname'] networks[network_id]['cidr_block'] = resource.get('extra_info', {}).get('cidr_block', '') networks[network_id]['region'] = resource['region'] networks[network_id]['kind'] = resource['vpc_type'].replace( 'shoot', 'dataplane') networks[network_id]['account_id'] = resource['account_id'] networks[network_id]['dbaas_info']['tenant_id'] = resource['tenant_id'] networks[network_id]['dbaas_info']['project_id'] = resource['project_id'] networks[network_id]['env'] = resource['env'] if networks[network_id]['env'] not in ['dev', 'staging', 'prod']: networks[network_id]['env'] = 'dev' networks[network_id]['dataplane_resources'].append({ 'kind': resource['product'], 'id': resource['rid'], 'name': resource['rname'], 'usedby': resource['usedby'], 'component': resource['component'], }) # pod_dict = {} # for pod in json.load(open('pod_node_scheduled_data.json', 'r')): # node_instance_id = pod['node_instance_id'] # if node_instance_id is not None: # pod_dict[node_instance_id.split('/')[-1]] = pod['pod'] # attach resource values for resource in resources: network_id = get_network_id(resource) if util.is_empty(network_id): continue if network_id not in networks: continue # if resource['vpc_id'] == 'null' and resource['product'] not in [AWSProduct.EBS, GCPProduct.Disk]: # continue # if resource['product'] == 'ebs' and resource['vpc_name'] != 'null': # for id, network in networks.items(): # if network['vpc_name'] == resource['vpc_name']: # resource['vpc_id'] = id # break # if resource['product'] in {AWSProduct.EBS, GCPProduct.Disk} and resource['vpc_id'] == 'null': # continue # if resource['vpc_id'] not in networks: # continue if resource['cluster_id'] in ['null', 'k8s-infra', 'admin']: result = { 'kind': resource['product'], 'id': resource['rid'], 'name': resource['rname'], 'usedby': resource['usedby'], 'component': resource['component'], } if resource['product'] in {AWSProduct.NAT, GCPProduct.NAT}: result.update({'nat_public_ip': resource['extra_info']['public_ip']}) networks[network_id]['dataplane_resources'].append(result) else: cluster_id = resource['cluster_id'] # initialize cluster info dict if cluster_id not in networks[network_id]['clusters']: networks[network_id]['clusters'][cluster_id] = { 'cluster_id': cluster_id, 'name': '', 'dbaas_info_status': '', 'infra_status': '', 'layout': '', 'resources': [] } item = { 'kind': resource['product'], 'id': resource['rid'], 'name': resource['rname'], 'usedby': resource['usedby'], 'component': resource['component'], 'cluster_id': cluster_id, } networks[network_id]['clusters'][cluster_id]['resources'].append(item) if resource['product'] in {AWSProduct.EBS, GCPProduct.Disk}: # matched = TIDB_POD_NAME_IN_EBS_PVC_NAME_RE.search(resource.get('ebs_pvc_name', '')) # if matched is not None: # item['pod_name'] = matched.group() item.update({ 'pod_name': resource['member'], 'ebs_type': resource['extra_info'].get('ebs_type', ''), 'ebs_pvc_name': resource['extra_info']['ebs_pvc_name'], 'ebs_ec2_mount': resource['extra_info']['ebs_ec2_mount'], 'ebs_size': resource['extra_info'].get('ebs_size', ''), 'ebs_iops': resource['extra_info'].get('ebs_iops', ''), 'ebs_throughput': resource['extra_info'].get('ebs_throughput', ''), }) elif resource['product'] in {AWSProduct.EC2, GCPProduct.VM}: # pod_name = '' # if resource['id'] in pod_dict: # pod_name = pod_dict[resource['id']] item.update({ 'hostname': resource['extra_info']['hostname'], 'pod_name': resource['member'], }) elif resource['product'] in {AWSProduct.SG}: pass else: pass # attach dbaas cluster info to network cluster_info = pd.read_pickle( CLUSTER_TENANT_INFO_PKL_FILE, compression="gzip") for _, network in networks.items(): network_id = get_network_id(network) matching_clusters = cluster_info[(cluster_info['tenant_id'] == network['dbaas_info']['tenant_id']) & (cluster_info['project_id'] == network['dbaas_info']['project_id']) & (cluster_info['region'] == network['region']) & (~cluster_info['cluster_status'].isin(['cleared', 'deletecomplete']))] if network['dbaas_info']['tenant_id'] == 'null': # vpc resources label is nil, get info from metadb matching_clusters = cluster_info[(cluster_info['region'] == network['region']) & (~cluster_info['cluster_status'].isin(['cleared', 'deletecomplete'])) & (cluster_info['cluster_id'].isin(network['clusters'].keys()))] if not matching_clusters.empty: for cluster in matching_clusters.iloc: if network['dbaas_info']['tenant_id'] == 'null': network['dbaas_info'].update({ 'tenant_id': cluster['tenant_id'], 'project_id': cluster['project_id'], }) network['dbaas_info'].update({ 'tenant_name': cluster['tenant_name'], 'project_name': cluster['project_name'], 'tenant_kind': cluster['tenant_kind'], }) if cluster['cluster_id'] not in networks[network_id]['clusters']: network['clusters'][cluster['cluster_id']] = { 'cluster_id': cluster['cluster_id'], 'name': cluster['cluster_name'], 'dbaas_info_status': cluster['cluster_status'], 'infra_status': '', 'layout': cluster['layout'], 'resources': [] } else: network['clusters'][cluster['cluster_id']].update({ 'cluster_id': cluster['cluster_id'], 'name': cluster['cluster_name'], 'dbaas_info_status': cluster['cluster_status'], 'layout': cluster['layout'] }) # analysis infra and k8s resources infra_network_dict = {} for network in json.load(open(INFRA_EXPORTER_NETWORK_ASSET_SNAPSHOT_ANALYSIS_RESULT, 'r')): network['migrated_network_name'] = '' if 'aws-' in network['network_name'] or 'gcp-' in network['network_name']: # gardener network network['migrated_network_name'] = network['network_name'] network['network'] = network['network_name'].replace( 'aws-', 'eks-').replace('gcp-', 'gke-') infra_network_dict[(network['env'], network['tenant'], network['project'], network['region'])] = network for _, network in networks.items(): infra_network = infra_network_dict.get( (network['env'], network['dbaas_info']['tenant_id'], network['dbaas_info']['project_id'], network['region'])) if infra_network is None: continue network['status'] = infra_network['status'] network['infra_resources'] = { 'namespace': infra_network['namespace'], 'network': infra_network['network_name'], 'vpc_peering': infra_network['networkpeer_list'], 'migrated_network': infra_network['migrated_network_name'], } for cluster_id, cluster_status in infra_network['cluster_list'].items(): if cluster_id in network['clusters']: network['clusters'][cluster_id]['infra_status'] = cluster_status # TODO: add backup, tiflow job, DM job, etc. # analysis features(problems) of the network for _, network in networks.items(): # 3-layer network if 'dedicated-' in network['vpc_name']: network['features'].append('3_layer_dedicated_network') # TODO: support 3-layer infra-exporter to unify the information continue # 1. find more than 3 nats nat_count = 0 for resource in network['dataplane_resources']: if resource['kind'] == 'nat': nat_count += 1 if nat_count > 3: network['features'].append('more_than_3_nats') # if network status is not normal, or creating, nat should be 0 if network['status'] not in ['Normal', 'Creating'] and nat_count != 0: network['features'].append('nat_not_zero_for_normal_network') # if network status is not normal, or Hibernated if network['status'] not in ['Normal', 'Hibernated']: network['features'].append('network_status_not_normal') # retired gardener network meta exists if network['infra_resources']['migrated_network'] != '': network['features'].append('retired_gardener_network_meta_exists') # if cluster status is not running for cluster in list(network['clusters'].values()): if cluster['infra_status'] not in ['Running', 'Suspend']: network['features'].append('cluster_status_not_running_or_suspend') break # network with vpc peering if len(network['infra_resources']['vpc_peering']) > 0: network['features'].append('vpc_peering_exists') # 3. find unused lb # 4. find unused ebs # 5. find unused ec2 # analysis network structure json.dump(list(networks.values()), open( TENANT_RESOURCE_UNITS_ANALYSIS_JSON_FILE, 'w'), indent=4) # find the problems in vpc