import json import csv import re from datetime import datetime clusters = json.load(open('./resources/cluster_resources.json'))['items'] networks = json.load(open('./resources/network_resources.json'))['items'] backups = json.load(open('./resources/backup_resources.json'))['items'] peerings = json.load(open('./resources/peering_resources.json'))['items'] id2tenant = {} with open('./resources/all_tenants.csv', newline='') as csvfile: spamreader = csv.reader(csvfile, delimiter=',', quotechar='|', ) for row in spamreader: id = row[0] id2tenant[id] = { 'id': id, 'name': row[1], 'kind': row[2], 'status': row[3], } network_key2network = {} results = {} not_migrated_networks = [] def new_network_value(): return { 'name': '', # network key 'networkDeletionTimestamp': '', # metadata.deletionTimestamp 'status': '', # status.phase 'tenant': '', # tenant id 'project': '', # project id 'clusters': [], 'backups': [], 'peerings': [], 'migrated_network': None, } def get_backup_region(backup) -> str: region_in_label = backup['metadata']['labels'].get('cloud.pingcap.com/region', '') if region_in_label != '': return region_in_label bucket_name = backup['status'].get('bucketName', '') if bucket_name == '': print('## migrated cluster\'s backup', backup['metadata']['labels']['tags.tidbcloud.com/cluster']) return 'unknown-region' project = backup['metadata']['labels']["tags.tidbcloud.com/project"] if project == '': return 'unknown-region' # bucket name like 'dbaas-prod-asia-northeast1-1351015692840734720' bucket_name = bucket_name.replace("dbaas-prod-", "") bucket_name = bucket_name.replace(f"-{project}", "") return bucket_name for network in networks: networkKey = network['metadata']['namespace'] + '/' + network['metadata']['name'] network_key2network[networkKey] = network networkDeletionTimestamp = '' if 'deletionTimestamp' in network['metadata']: networkDeletionTimestamp = 'deleted:' + network['metadata']['deletionTimestamp'] if networkKey not in results: results[networkKey] = new_network_value() results[networkKey]['name'] = networkKey results[networkKey]['networkDeletionTimestamp'] = networkDeletionTimestamp results[networkKey]['status'] = network['status']['phase'] tenant_id = network['metadata']['labels']['tags.tidbcloud.com/tenant'] results[networkKey]['tenant'] = tenant_id results[networkKey]['tenant_name'] = id2tenant.get(tenant_id, {}).get('name', 'unknown') results[networkKey]['tenant_kind'] = id2tenant.get(tenant_id, {}).get('kind', 'unknown') results[networkKey]['tenant_status'] = id2tenant.get(tenant_id, {}).get('status', 'unknown') results[networkKey]['project'] = network['metadata']['labels']['tags.tidbcloud.com/project'] for cluster in clusters: networkKey = cluster['metadata']['namespace'] + '/' + cluster['spec']['networkName'] clusterDeletionTimestamp = '' if 'deletionTimestamp' in cluster['metadata']: clusterDeletionTimestamp = 'deleted:' + cluster['metadata']['deletionTimestamp'] cluster_result = {'name':cluster['metadata']['name'], 'deletionTimestamp':clusterDeletionTimestamp, 'status': cluster['status']['phase']} if networkKey not in results: results[networkKey] = new_network_value() results[networkKey]['clusters'].append(cluster_result) for backup in backups: if backup['status']['phase'] == "Failed" or backup['status']['phase'] == 'Prepare': continue if backup['metadata'].get('annotations', {}).get('tags.tidbcloud.com/migrated-from-central') == 'true': print('## migrated cluster\'s backup', backup['metadata']['labels']['tags.tidbcloud.com/cluster']) continue networkKey = (backup['metadata']['namespace'] + '/' + backup['metadata']['labels']['cloud.pingcap.com/provider-type'] + "-" + get_backup_region(backup)) backupDeletionTimestamp = '' if 'deletionTimestamp' in backup['metadata']: backupDeletionTimestamp = 'deleted:' + backup['metadata']['deletionTimestamp'] backup_result = {'name':backup['metadata']['name'], 'deletionTimestamp':backupDeletionTimestamp, 'status': backup['status']['phase']} if networkKey not in results: results[networkKey] = new_network_value() results[networkKey]['backups'].append(backup_result) for peering in peerings: networkKey = (peering['metadata']['namespace'] + '/' + peering['spec']['source']) peeringDeletionTimestamp = '' if 'deletionTimestamp' in peering['metadata']: peeringDeletionTimestamp = 'deleted:' + peering['metadata']['deletionTimestamp'] creationTimestamp = datetime.strptime(peering['metadata']['creationTimestamp'], "%Y-%m-%dT%H:%M:%SZ") now = datetime.utcnow() if peering['status']['phase'] == "Pending" and (now-creationTimestamp).total_seconds()>3600*24*3: continue peering_result = {'name':peering['metadata']['name'], 'status': peering['status']['phase']} if networkKey not in results: results[networkKey] = new_network_value() results[networkKey]['peerings'].append(peering_result) # match migrated network with gardener network for network in networks: networkKey = network['metadata']['namespace'] + '/' + network['metadata']['name'] if 'aws-' in networkKey or 'gcp-' in networkKey: migrated_network_key = networkKey.replace('aws-', 'eks-').replace('gcp-', 'gke-') if migrated_network_key not in results: not_migrated_networks.append({ 'name': networkKey, 'status': network['status']['phase'], 'body': results[networkKey], }) continue migrated_network_result = results[migrated_network_key] migrated_network = network_key2network[migrated_network_key] # new-arch networks is newly created rather than a migration of gardener network if len(migrated_network['metadata']['annotations'].get('cloud.pingcap.com/migrate', '')) == 0: not_migrated_networks.append({ 'name': networkKey, 'body': results[networkKey], 'migratedName': migrated_network_key, 'migrated_network': migrated_network_result, 'newlyCreated': True, }) continue migrated_network_info = { 'name': migrated_network_key, 'networkDeletionTimestamp': migrated_network_result['networkDeletionTimestamp'], 'status': migrated_network_result['status'], 'clusters': migrated_network_result['clusters'], 'backups': migrated_network_result['backups'], 'peerings': migrated_network_result['peerings'], } results[networkKey]['migrated_network'] = migrated_network_info # analyse result again deleting_networks = [] migrated_but_empty_networks = [] migrated_with_cluster = [] not_running_networks = [] migrated_networks = [] for k,v in results.items(): conclusion = '' isDeleting = False isMigratedButEmpty = False isMigratedWithCluster = False isNotRunning = False if v['networkDeletionTimestamp'] != '': isDeleting = True conclusion += 'Network is deleting. ' if v['status'] != 'Normal' and v['status'] != 'Hibernated': isNotRunning = True conclusion += 'Network is not running or hibernated. ' for cluster in v['clusters']: if cluster['status'] != 'Running' and cluster['status'] != 'Suspend': isNotRunning = True conclusion += 'Cluster ' + cluster['name'] + ' is not running or suspend. ' if cluster['deletionTimestamp'] != '': isDeleting = True conclusion += 'Cluster ' + cluster['name'] + ' is deleting. ' # migrated but no clusters migrated_network = v['migrated_network'] if migrated_network is not None: v['without_clusters'] = len(v['clusters']) == 0 and len(migrated_network['clusters']) == 0 v['gardener_without_clusters'] = len(v['clusters']) == 0 v['newarch_without_clusters'] = len(migrated_network['clusters']) == 0 v['without_backups'] = len(v['backups']) == 0 and len(migrated_network['backups']) == 0 v['gardener_without_backups'] = len(v['backups']) == 0 v['newarch_without_backups'] = len(migrated_network['backups']) == 0 v['without_peerings'] = len(v['peerings']) == 0 and len(migrated_network['peerings']) == 0 v['gardener_without_peerings'] = len(v['peerings']) == 0 v['newarch_without_peerings'] = len(migrated_network['peerings']) == 0 migrated_networks.append({'network': k, 'body': v}) conclusion += 'Network is migrated' if len(v['clusters']) == 0: isMigratedButEmpty = True conclusion += ' Without clusters ' # migrated with clusters if len(v['clusters']) > 0: isMigratedWithCluster = True conclusion += ' With clusters ' if isDeleting: deleting_networks.append({'network': k, 'conclusion': conclusion,'body':v}) elif isNotRunning: not_running_networks.append({'network': k, 'conclusion': conclusion,'body':v}) if isMigratedButEmpty: migrated_but_empty_networks.append({'network': k, 'conclusion': conclusion,'body':v}) if isMigratedWithCluster: migrated_with_cluster.append({'network': k, 'conclusion': conclusion,'body':v}) json.dump(deleting_networks, open('./results/deleting_networks.json', 'w'), indent=2) json.dump(not_running_networks, open('./results/not_running_networks.json', 'w'), indent=2) json.dump(migrated_but_empty_networks, open('./results/migrated_but_empty_networks.json', 'w'), indent=2) json.dump(migrated_with_cluster, open('./results/migrated_with_cluster.json', 'w'), indent=2) json.dump(results, open('./results/all_results.json', 'w'), indent=2) json.dump(migrated_networks, open('./results/migrated_networks.json', 'w'), indent=2) json.dump(not_migrated_networks, open('./results/not_migrated_networks.json', 'w'), indent=2) with open('./results/migrated_networks.csv', mode='w') as f: writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) writer.writerow(['tenant_id', 'tenant_name', 'tenant_kind', 'tenant_status', 'project_id', 'network_name', 'network_status', 'migrated_network_name', 'migrated_network_status', 'full_empty','gardener_without_clusters', 'newarch_without_clusters', 'gardener_without_backups', 'newarch_without_backups', 'gardener_without_peerings', 'newarch_without_peerings', 'clusters', 'backups', 'peerings' ]) for migrated_network in migrated_networks: body = migrated_network['body'] cluster_names = [] for cluster in body['clusters']: cluster_names.append(cluster['name']) for cluster in body['migrated_network']['clusters']: cluster_names.append(cluster['name']) backup_names = [] for backup in body['backups']: backup_names.append(backup['name']) for backup in body['migrated_network']['backups']: backup_names.append(backup['name']) peering_names = [] for peering in body['peerings']: peering_names.append(peering['name']) for peering in body['migrated_network']['peerings']: peering_names.append(peering['name']) full_empty = body['without_clusters'] and body['without_backups'] and body['without_peerings'] writer.writerow([ f"'{body['tenant']}'", body['tenant_name'].replace(',', '_'), body['tenant_kind'], body['tenant_status'], f"'{body['project']}'", body['name'], body['status'], body['migrated_network']['name'], body['migrated_network']['status'], full_empty, body['gardener_without_clusters'], body['newarch_without_clusters'], body['gardener_without_backups'], body['newarch_without_backups'], body['gardener_without_peerings'], body['newarch_without_peerings'], cluster_names.__str__(), backup_names.__str__(), peering_names.__str__(), ]) with open('./results/not_migrated_networks.csv', mode='w') as f: writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) writer.writerow(['tenant_id', 'tenant_name', 'tenant_kind', 'tenant_status', 'project_id', 'network_name', 'network_status', 'migrated_network_name', 'migrated_network_status', 'full_empty', 'newarch_is_newly_created', 'gardener_without_clusters', 'gardener_without_backups', 'gardener_without_peerings', 'clusters', 'backups', 'peerings' ]) for network in not_migrated_networks: body = network['body'] cluster_names = [] for cluster in body['clusters']: cluster_names.append(cluster['name']) backup_names = [] for backup in body['backups']: backup_names.append(backup['name']) peering_names = [] for peering in body['peerings']: peering_names.append(peering['name']) gardener_without_clusters = len(cluster_names) == 0 gardener_without_backups = len(backup_names) == 0 gardener_without_peerings = len(peering_names) == 0 full_empty = gardener_without_clusters and gardener_without_backups and gardener_without_peerings writer.writerow([ f"'{body['tenant']}'", body['tenant_name'].replace(',', '_'), body['tenant_kind'], body['tenant_status'], f"'{body['project']}'", body['name'], body['status'], network.get('migrated_network',{}).get('name', 'none'), network.get('migrated_network', {}).get('status', 'non'), full_empty, network.get('newlyCreated', False), gardener_without_clusters, gardener_without_backups, gardener_without_peerings, cluster_names.__str__(), backup_names.__str__(), peering_names.__str__(), ])