import dataclasses import logging from datetime import datetime import boto3 import botocore.client from botocore.exceptions import ClientError from analyst.libs import consts from analyst.libs.consts import UsedBy from analyst.resource_leak import model from analyst.resource_leak.model import CloudAsset from actor.recycler.cloud_resource_recycler.base import CloudResourceCleaner, CleanResult logger = logging.getLogger("recycler.cloud_resource_recycle") # AWS SDK boto3: https://boto3.amazonaws.com/v1/documentation/api/latest/index.html class AWSCleaner(CloudResourceCleaner): def __init__(self, dryrun: bool = False): super().__init__(dryrun) def get_aws_session(self, account_id: str) -> boto3.Session: return boto3.Session(profile_name=account_id) class AWSEC2Cleaner(AWSCleaner): USE_API_DRY_RUN = True def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: try: ec2_client = self.get_aws_session(r.account_id).client('ec2', region_name=r.region) response = ec2_client.terminate_instances(InstanceIds=[r.rid], DryRun=self.dryrun) if response['TerminatingInstances'][0]['CurrentState']['Name'] in ['shutting-down', 'terminated']: return CleanResult(result=consts.CleanResult.Cleaned, reason=f'EC2 instance {r.rid} successfully terminated') else: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to terminate EC2 instance {r.rid}. Failure reason: {response["TerminatingInstances"][0]["CurrentState"]["Name"]}') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error terminating EC2 instance {r.rid}: {str(e)}') # TODO(chenfei): check network status before clean class AWSNATCleaner(AWSCleaner): USE_API_DRY_RUN = True def _clean_metadata_from_k8s(self, r: model.CloudAsset): # Delete Crossplane CR from k8s apiserver if self.cr_api and r.rname: try: self.cr_api.delete_cluster_custom_object( group="ec2.aws.crossplane.io", version="v1beta1", namespace="default", plural="natgateways", name=r.rname ) except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting Crossplane CR for EKS cluster {r.rname}: {str(e)}') def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: # follow https://repost.aws/knowledge-center/vpc-delete-nat-gateway, we have to manually release the EIP after # deleting the NAT Gateway eip_allocation_id = r.extra_info.get('eip_allocation_id', '') try: ec2_client = self.get_aws_session(r.account_id).client('ec2', region_name=r.region) # retrieve the allocation id of the EIP associated with the NAT Gateway if not eip_allocation_id: nat_gateways = ec2_client.describe_nat_gateways(NatGatewayIds=[r.rid]) if nat_gateways['NatGateways']: eip_allocation_id = nat_gateways['NatGateways'][0]['NatGatewayAddresses'][0]['AllocationId'] # Delete NAT Gateway nat_response = ec2_client.delete_nat_gateway(NatGatewayId=r.rid, DryRun=self.dryrun) if nat_response['ResponseMetadata']['HTTPStatusCode'] != 200: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete NAT Gateway {r.rid}') # Delete EIP self._delete_eip(ec2_client, r, eip_allocation_id) return CleanResult(result=consts.CleanResult.Cleaned, reason=f'NAT Gateway {r.rid} and associated EIP {eip_allocation_id} successfully deleted') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting NAT Gateway {r.rid} and associated EIP {eip_allocation_id}: {str(e)}, tp: {type(e)}') def _delete_eip(self, ec2_client: botocore.client.BaseClient, r: CloudAsset, eip_allocation_id: str): # Release the associated EIP try: eip_response = ec2_client.release_address(AllocationId=eip_allocation_id, DryRun=self.dryrun) if eip_response['ResponseMetadata']['HTTPStatusCode'] != 200: raise Exception( f'delete NAT associated EIP return non 200: {eip_response}') except botocore.exceptions.ClientError as e: err_code = e.response.get('Error', {}).get('Code') logger.info( f'Error deleting NAT associated EIP {eip_allocation_id}: {str(e)}, tp: {type(e)}, code: {err_code}') if err_code == 'InvalidAllocationID.NotFound': logger.info('ignore because error is NotFound') return raise class AWSELBv1Cleaner(AWSCleaner): def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: try: elb_client = self.get_aws_session(r.account_id).client('elb', region_name=r.region) try: # check target groups health_instances = [] instances = elb_client.describe_instance_health(LoadBalancerName=r.rid) if instances['InstanceStates']: for instance in instances['InstanceStates']: # InService | OutOfService | Unknown if instance['State'] in {'InService'}: health_instances.append(instance) # CLB now only used by gardener cluster and CLBs used by shoot belong to control-plane # So that deletion is safe even if there are health instances in target of shoot CLB if health_instances and r.usedby not in {UsedBy.DBaaSControlPlaneShoot}: return CleanResult(result=consts.CleanResult.Failed, reason=f'ELBv1 {r.rid} still has health instances: {health_instances}, deletion cancelled') except ClientError as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Deletion cancelled. Unable to get ELBv1 {r.rid} information: {e}') response = elb_client.delete_load_balancer(LoadBalancerName=r.rname) if response['ResponseMetadata']['HTTPStatusCode'] == 200: return CleanResult(result=consts.CleanResult.Cleaned, reason=f'ELBv1 {r.rname} successfully deleted') else: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete ELBv1 {r.rname}') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting ELBv1 {r.rname}: {str(e)}') class AWSELBv2Cleaner(AWSCleaner): def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: arn = r.extra_info.get('arn', '') try: elbv2_client = self.get_aws_session(r.account_id).client('elbv2', region_name=r.region) try: # check target groups health_targets = [] target_groups = elbv2_client.describe_target_groups(LoadBalancerArn=arn) if target_groups['TargetGroups']: for target_group in target_groups['TargetGroups']: target_health = elbv2_client.describe_target_health( TargetGroupArn=target_group['TargetGroupArn']) for target in target_health['TargetHealthDescriptions'] or []: # 'State': 'initial'|'healthy'|'unhealthy'|'unhealthy.draining'|'unused'|'draining'|'unavailable', if target['TargetHealth']['State'] in {'initial', 'health'}: health_targets.append(target) if health_targets: return CleanResult(result=consts.CleanResult.Failed, reason=f'ELBv2 {r.rid} still has health targets: {health_targets}') else: logger.info(f'NLB {r.rid} has target groups, but they are all empty') listeners = elbv2_client.describe_listeners(LoadBalancerArn=arn) if listeners['Listeners']: logger.debug(f'NLB {r.rid} has listeners: {listeners["Listeners"]}') # todo(chenfei): to find out all dependents, like VPC Endpoint Service except ClientError as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Deletion cancelled. Unable to get NLB {r.rid} information: {e}') response = elbv2_client.delete_load_balancer(LoadBalancerArn=arn) if response['ResponseMetadata']['HTTPStatusCode'] == 200: return CleanResult(result=consts.CleanResult.Cleaned, reason=f'NLB {r.rid} successfully deleted') else: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete NLB {r.rid}') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting NLB {r.rid}: {str(e)}') class AWSEKSCleaner(AWSCleaner): def _clean_metadata_from_k8s(self, r: model.CloudAsset): # Delete Crossplane CR from k8s apiserver if self.cr_api and r.rname: try: self.cr_api.delete_cluster_custom_object( group="eks.aws.crossplane.io", version="v1beta1", namespace="default", plural="clusters", name=r.rname ) except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting Crossplane CR for EKS cluster {r.rname}: {str(e)}') def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: try: eks_client = self.get_aws_session(r.account_id).client('eks', region_name=r.region) try: cluster = eks_client.describe_cluster(name=r.rname) if cluster['cluster']['status'] != 'ACTIVE': return CleanResult(result=consts.CleanResult.Failed, reason=f'EKS cluster {r.rname} status is not ACTIVE, deletion cancelled') nodegroups = eks_client.list_nodegroups(clusterName=r.rname) if nodegroups['nodegroups']: return CleanResult(result=consts.CleanResult.Failed, reason=f'EKS cluster {r.rname} still has node groups, deletion cancelled') fargate_profiles = eks_client.list_fargate_profiles(clusterName=r.rname) if fargate_profiles['fargateProfileNames']: return CleanResult(result=consts.CleanResult.Failed, reason=f'EKS cluster {r.rname} still has Fargate profiles, deletion cancelled') except ClientError: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to get EKS cluster {r.rname} information, deletion cancelled') # Delete EKS cluster response = eks_client.delete_cluster(name=r.rname) if response['ResponseMetadata']['HTTPStatusCode'] != 200: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete EKS cluster {r.rname}') return CleanResult(result=consts.CleanResult.Cleaned, reason=f'EKS cluster {r.rname} and associated Crossplane CR successfully deleted') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting EKS cluster {r.rname}: {str(e)}') class AWSEBSCleaner(AWSCleaner): USE_API_DRY_RUN = True def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: try: ec2_client = self.get_aws_session(r.account_id).client('ec2', region_name=r.region) try: volume = ec2_client.describe_volumes(VolumeIds=[r.rid])['Volumes'][0] if volume['State'] != 'available': return CleanResult(result=consts.CleanResult.Failed, reason=f'EBS volume {r.rid} status is not available, deletion cancelled') snapshots = ec2_client.describe_snapshots(Filters=[{'Name': 'volume-id', 'Values': [r.rid]}]) if snapshots['Snapshots']: return CleanResult(result=consts.CleanResult.Failed, reason=f'EBS volume {r.rid} still has associated snapshots, deletion cancelled') if volume['Attachments']: return CleanResult(result=consts.CleanResult.Failed, reason=f'EBS volume {r.rid} still has attached instances, deletion cancelled') except ClientError: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to get EBS volume {r.rid} information, deletion cancelled') response = ec2_client.delete_volume(VolumeId=r.rid, DryRun=self.dryrun) if response['ResponseMetadata']['HTTPStatusCode'] == 200: return CleanResult(result=consts.CleanResult.Cleaned, reason=f'EBS volume {r.rid} successfully deleted') else: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete EBS volume {r.rid}') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting EBS volume {r.rid}: {str(e)}') class AWSAutoScalingGroupCleaner(AWSCleaner): USE_API_DRY_RUN = False def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: try: asg_client = self.get_aws_session(r.account_id).client('autoscaling', region_name=r.region) try: asg = asg_client.describe_auto_scaling_groups(AutoScalingGroupNames=[r.rname])['AutoScalingGroups'][0] if asg['Instances']: return CleanResult(result=consts.CleanResult.Failed, reason=f'AutoScalingGroup {r.rname} still has running instances: {asg["Instances"]}, deletion cancelled') except ClientError: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to get AutoScalingGroup {r.rname} information, deletion cancelled') response = asg_client.delete_auto_scaling_group( AutoScalingGroupName=r.rname, # Specifies that the group is to be deleted along with all instances associated with the group, # without waiting for all instances to be terminated. # This action also deletes any outstanding lifecycle actions associated with the group. ForceDelete=False) if response['ResponseMetadata']['HTTPStatusCode'] == 200: return CleanResult(result=consts.CleanResult.Cleaned, reason=f'AutoScalingGroup {r.rname} successfully deleted') else: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete AutoScalingGroup {r.rname}') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting AutoScalingGroup {r.rname}: {str(e)}') class AWSEKSNodeGroupCleaner(AWSCleaner): USE_API_DRY_RUN = False SKIP_NON_EMPTY_NODE_GROUP = False def _clean_cloud_resource(self, r: model.CloudAsset) -> CleanResult: try: eks_client = self.get_aws_session(r.account_id).client('eks', region_name=r.region) try: node_group = eks_client.describe_nodegroup(clusterName=r.k8s_name, nodegroupName=r.rname)['nodegroup'] # Check if there are any nodes in the node group if node_group['scalingConfig']['desiredSize'] > 0: if self.SKIP_NON_EMPTY_NODE_GROUP: return CleanResult(result=consts.CleanResult.Failed, reason=f'EKS Node Group {r.rname} still has {node_group["scalingConfig"]["desiredSize"]} nodes, deletion cancelled') else: logger.warning( f'EKS Node Group {r.rname} still has {node_group["scalingConfig"]["desiredSize"]} nodes') except ClientError: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to get EKS Node Group {r.rname} information, deletion cancelled') # delete node group response = eks_client.delete_nodegroup(clusterName=r.k8s_name, nodegroupName=r.rname) if response['ResponseMetadata']['HTTPStatusCode'] == 200: return CleanResult(result=consts.CleanResult.Cleaned, reason=f'EKS Node Group {r.rname} successfully deleted') else: return CleanResult(result=consts.CleanResult.Failed, reason=f'Unable to delete EKS Node Group {r.rname}') except Exception as e: return CleanResult(result=consts.CleanResult.Failed, reason=f'Error deleting EKS Node Group {r.rname}: {str(e)}')