# design: https://pingcap.feishu.cn/wiki/wikcnsBs5hwaBxiWUVQ22DHMbHe import copy import difflib import json import pprint import dictdiffer resource_template = { 'product': 'inter_key', 'resource_id': 'inter_key', 'region': 'inter_key', 'environment': '', 'servicetype': '', 'usedby': '', 'component': '', 'tenant': '', 'project': '', 'cluster': '' } analysis_result_mapping = { 'environment': 'env', 'tenant': 'tenant_id', 'cluster': 'cluster_id', 'project': 'project_id', 'servicetype': 'service_type', 'usedby': 'usedby', 'component': 'component', 'product': 'product' } resource_id_mapping = { 'vm': 'name', 'disk': 'name', } resources_labels_status = [] valid_resources = [] need_sync_resources = [] invalid_resources_labels = [] apply_labels_task_list = [] diff_apply_labels_list = [] def compare_dicts(d1, d2): return ('\n' + '\n'.join(difflib.ndiff( pprint.pformat(d1).splitlines(), pprint.pformat(d2).splitlines()))) def get_label_status(resource, resource_key): resource_label_status = copy.deepcopy(resource_template) resource_label_status.update({ 'product': resource_key, 'region': resource['region'], 'resource_id': resource[resource_id_mapping[resource_key]], 'account_id': resource['account_id'], 'labelFingerprint': resource['labelFingerprint'], 'existed_labels': json.loads(resource['labels']), }) if resource['labels'] == 'null': labels = {} else: labels = json.loads(resource['labels']) for tag in labels.items(): if tag[0] in resource_label_status.keys(): resource_label_status[tag[0]] = tag[1] return resource_label_status def get_label_spec(resource, resource_key): resource_label_spec = copy.deepcopy(resource_template) resource_label_spec.update({ 'product': resource_key, 'region': resource['region'], 'account_id': resource['account_id'], 'resource_id': resource[resource_id_mapping[resource_key]], 'labelFingerprint': resource['labelFingerprint'], 'existed_labels': json.loads(resource['labels']), }) for k, v in analysis_result_mapping.items(): if v in resource: if resource[v] == 'null': resource[v] = '' resource_label_spec[k] = resource[v] if resource_label_spec['account_id'] in 'dbaas-prod': resource_label_spec['environment'] = 'prod' return resource_label_spec def make_gcp_resource_label_spec(resource_label_spec): labels = resource_label_spec['existed_labels'] labels.update(resource_label_spec) labels.pop('product') labels.pop('resource_id') labels.pop('account_id') labels.pop('region') labels.pop('existed_labels') labels.pop('labelFingerprint') if 'name' in labels.keys(): labels.pop('name') return labels def make_apply_labels_list(resource_label_spec, resource_label_status): # pass serverless resources and devtier resources if resource_label_spec['servicetype'] in ['devtier', 'serverless']: return if resource_label_status['servicetype'] in ['devtier', 'serverless']: return if resource_label_status['component'] != 'extra-tidb' and resource_label_status['usedby'] == 'customer-tidb' and \ resource_label_spec['usedby'] != 'customer-tidb': # print('usedby changes, status:%s, spec:%s' % # (resource_label_status, resource_label_spec)) # print(compare_dicts(resource_label_status, resource_label_spec)) return if resource_label_status['component'] == 'tidb-lightning': # TODO: fix tidb-lightning cluster issue return # if the current tag is more detailed, keep it if resource_label_spec['cluster'] == 'admin' and resource_label_status['cluster'] != 'admin' and \ resource_label_status['cluster'] != '': resource_label_spec['cluster'] = resource_label_status['cluster'] if resource_label_spec['component'] == 'csi' and resource_label_status['component'] != 'csi' and \ resource_label_status['component'] != '': resource_label_spec['component'] = resource_label_status['component'] if resource_label_spec['usedby'] == '' and resource_label_status['usedby'] == 'customer-tidb': resource_label_spec['usedby'] = resource_label_status['usedby'] if resource_label_spec['usedby'] in ['dbaas-control-plane-shoot', 'dbaas-control-plane-seed', 'dbaas-control-plane-base'] and resource_label_status['component'] == '': resource_label_spec['component'] = resource_label_spec['usedby'] # if resource_label spec key is empty, we record it in invalid_resources_labels for key in resource_label_spec.keys(): if key not in ['product', 'resource_id', 'region', 'cluster'] and resource_label_spec[key] == '': # there's empty tag in spec, we need to sync # if the status is more detailed, we just ignore it. # print('fixing failed, status:%s, spec:%s' % # (resource_label_status, resource_label_spec)) # print(compare_dicts(resource_label_status, resource_label_spec)) invalid_resources_labels.append( {'spec': resource_label_spec, 'status': resource_label_status}) return # check if the existed label changed for k, v in resource_label_status.items(): if v != '' and v != resource_label_spec[k]: # print('validate resources changed status:%s, spec:%s' % # (resource_label_status, resource_label_spec)) # print(compare_dicts(resource_label_status, resource_label_spec)) return is_label_changed = False for k, v in resource_label_spec.items(): if k in resource_label_status and v != resource_label_status[k]: # print('existed label changed status:%s, spec:%s' % # (resource_label_status, resource_label_spec)) # print(compare_dicts(resource_label_status, resource_label_spec)) is_label_changed = True break if k not in resource_label_status: # print('new label added status:%s, spec:%s' % # (resource_label_status, resource_label_spec)) # print(compare_dicts(resource_label_status, resource_label_spec)) is_label_changed = True break if is_label_changed: diff_apply_labels_list.append( {'spec': resource_label_spec, 'status': resource_label_status}) apply_labels_task_list.append( { 'resource_id': resource_label_spec['resource_id'], 'labelFingerprint': resource_label_spec['labelFingerprint'], 'account_id': resource_label_spec['account_id'], "product": resource_label_spec['product'], 'region': resource_label_spec['region'].split('zones/')[-1], 'labels': make_gcp_resource_label_spec( resource_label_spec) }) return # valid resource valid_resources.append( {'spec': resource_label_spec, 'status': resource_label_status}) def sync_resources(asset_snapshot_analysised): for resource in asset_snapshot_analysised: resource_key = resource['product'] if resource_key not in resource_id_mapping.keys(): continue resource_label_status = get_label_status(resource, resource_key) resource_label_spec = get_label_spec(resource, resource_key) resources_labels_status.append(resource_label_status) make_apply_labels_list(resource_label_spec, resource_label_status) if __name__ == "__main__": asset_snapshot_analysised = json.load( open("gcp_asset_snapshot_analysis_result.json", 'r')) sync_resources(asset_snapshot_analysised) # dump_result json.dump(resources_labels_status, open( 'labels_gcp_resources_labels_status.json', 'w'), indent=4) json.dump(valid_resources, open( 'labels_gcp_valid_resources_labels.json', 'w'), indent=4) json.dump(invalid_resources_labels, open( 'labels_gcp_invalid_resources_labels.json', 'w'), indent=4) json.dump(apply_labels_task_list, open( 'labels_gcp_apply_labels_list.json', 'w'), indent=4) print( f'valid={len(valid_resources)}, invalid={len(invalid_resources_labels)}, apply={len(apply_labels_task_list)}') print(f'apply diff:') for each in diff_apply_labels_list: print(f'{each["spec"]["product"]}/{each["spec"]["resource_id"]}', list(dictdiffer.diff(each['status'], each['spec']))) # dump diff result to file with open('labels_gcp_apply_diff.diff', 'w') as f: for result in diff_apply_labels_list: for k, v in result['spec'].items(): if v != '' and v != result['status'][k]: print('apply list, status:%s, spec:%s' % (result['status'], result['spec']), file=f) print(compare_dicts( result['status'], result['spec']), file=f) break