import json def analysis_network_data(network_data): network_list = [] for network in network_data: network_detail = network['metric'] account, tenant, project = '', '', '' if 'free-tier' in network_detail['exported_provider']: continue if 'account' in network_detail: account = network_detail['account'] else: print("error: no account field in network_detail") if 'tenant_id' in network_detail: tenant = network_detail['tenant_id'] else: print("error: no tenant_id field in network_detail") if 'project_id' in network_detail: project = network_detail['project_id'] else: print("error: no project_id field in network_detail") network_result = { 'tenant': tenant, 'project': project, 'namespace': network_detail['exported_namespace'], 'network_name': network_detail['name'], 'status': '', # this is used for clould-native network cr 'dataplane': network_detail.get("dataplane") or "", 'region': network_detail['region'], 'account_id': account, 'cluster_list': {}, 'networkpeer_list': [], 'env': network_detail['env'], 'shoot_name': '', 'vpc_status': network_detail.get('vpc_status') or '', 'internet_status': network_detail.get('internet_status') or '', 'appbox_status': network_detail.get('appbox_status') or '', 'approute_status': network_detail['approutes'], } network_list.append(network_result) print("[INFO] total networks:", len(network_list)) return network_list def analysis_shoot_data(shoot_list, network_list): for network in network_list: if 'network_name' not in network or 'eks' in network['network_name'] or 'gke' in network['network_name']: continue for shoot in shoot_list['data']['result']: shoot_detail = shoot['metric'] if 'tenant' not in shoot_detail: continue if shoot_detail['tenant'] == network['tenant'] and shoot_detail['project'] == network['project'] and \ shoot_detail['exported_region'] == network['region']: network['shoot_name'] = shoot_detail['name'] return network_list def analysis_cluster_data(cluster_data, network_list): cnt = 0 cluster_cnt = 0 for cluster in cluster_data['data']['result']: cluster_detail = cluster['metric'] # eks "base/eks/us-west-2" has been paused if "base/eks/us-west-2" == cluster_detail.get('cluster', ''): continue env = cluster_detail['exported_namespace'].split( '-')[0].replace('nightly', 'dev') for personal_namespace in ['zl', 'yifan', 'zhenchi', 'shiori', 'rajesh', 'lc', 'jiajun', 'e2e', 'dingshun', 'chenshuning', 'chenfei']: if personal_namespace in cluster_detail['exported_namespace']: env = 'dev' cluster_cnt += 1 for network in network_list: if 'dataplane' not in cluster_detail or 'dataplane' not in network: continue if cluster_detail['dataplane'] == network['dataplane']: if 'status' in cluster_detail: network['cluster_list'].update( {cluster_detail['name']: cluster_detail['status']}) else: network['cluster_list'].update( {cluster_detail['name']: ''}) cnt += 1 print(f"[INFO] total cluster count: ", cluster_cnt) print(f"[INFO] clusters have been mapped to networks: ", cnt) return network_list def analysis_networkpeer_data(networkpeer_data, network_list): for networkpeer in networkpeer_data['data']['result']: networkpeer_detail = networkpeer['metric'] for network in network_list: if network['namespace'] == networkpeer_detail['exported_namespace'] and network['network_name'] == \ networkpeer_detail['source']: network['networkpeer_list'].append( networkpeer_detail['name']) return network_list def check_network_cluaster_mapping(network_list): print("[INFO] total networks:", len(network_list)) print("[INFO] do checking:", len(network_list)) for network in network_list: cluster_map = network.get('cluster_list') or {} approute_map = network.get('approute_status') or {} cluster_ids = set(cluster_map.keys()) if isinstance(cluster_map, dict) else set(cluster_map) approute_ids = set(approute_map.keys()) if isinstance(approute_map, dict) else set(approute_map) if cluster_ids or approute_ids: if len(cluster_ids) != len(approute_ids): print("warning: cluster/approute count mismatch", { 'dataplane': network.get('dataplane'), 'cluster_count': len(cluster_ids), 'approute_count': len(approute_ids), }) unequal_ids = cluster_ids.symmetric_difference(approute_ids) if unequal_ids: print("warning: cluster/approute id mismatch", { 'dataplane': network.get('dataplane'), 'approutes': approute_map.keys(), 'cluster': cluster_map.keys(), }) def analysis_pvc_data(pvc_data): pvc_list = {} for pvc in pvc_data['data']['result']: pvc_detail = pvc['metric'] if 'volumename' not in pvc_detail: continue pvc_list[pvc_detail['volumename']] = { 'pvc_name': pvc_detail['persistentvolumeclaim'], 'namespace': pvc_detail['namespace'], } return pvc_list if '__main__' == __name__: network_data = json.load( open('networks.json', 'r')) network_list = analysis_network_data(network_data) cluster_data = json.load( open('infra_exporter_cluster_asset_snapshot.json', 'r')) network_list = analysis_cluster_data(cluster_data, network_list) # TODO(chenfei): collect networkpeers in infra-exporter # networkpeer_data = json.load( # open('infra_api_resources_networkpeer_info_asset_snapshot.json', 'r')) # network_list = analysis_networkpeer_data(networkpeer_data, network_list) json.dump(network_list, open('new_network_result.json', 'w')) pvc_data = json.load( open('kube_persistentvolumeclaim_info_asset_snapshot.json', 'r')) pvc_list = analysis_pvc_data(pvc_data) json.dump(pvc_list, open( 'kube_persistentvolumeclaim_info_asset_snapshot_analysis_result.json', 'w')) # do checking check_network_cluaster_mapping(json.load(open('new_network_result.json', 'r')))