import logging from datetime import timedelta import boto3 from celery import shared_task from django.utils import timezone from labs.models import LabInstance as DBLabInstance logger = logging.getLogger(__name__) AWS_REGIONS = [ "us-west-2", "ap-northeast-1", "ap-southeast-1", "eu-central-1", ] @shared_task def aws_cloudformation_housekeeping(): """ Fallback cleanup task for AWS CloudFormation stacks. Purpose: - Acts as a safety net when a Lab Instance is not properly destroyed. - Scans configured regions and cleans up orphaned or expired stacks associated with lab instances. - Prevents unused resources from lingering and incurring unnecessary cost. Behavior: - Only targets stacks linked to a lab instance whose scheduled_terminate_at is in the past. - CREATE_COMPLETE or CREATE_FAILED: request deletion. - DELETE_FAILED: force deletion. - DELETE_IN_PROGRESS: skip. """ for region in AWS_REGIONS: _aws_cloudformation_housekeeping(region) def _aws_cloudformation_housekeeping(region: str): client = boto3.client("cloudformation", region_name=region) status_filter = [ "CREATE_COMPLETE", "CREATE_FAILED", "DELETE_FAILED", ] paginator = client.get_paginator("list_stacks") for page in paginator.paginate(StackStatusFilter=status_filter): for stack in page.get("StackSummaries", []): stack_name = stack["StackName"] stack_status = stack["StackStatus"] try: DBLabInstance.objects.get( unique_tag=stack_name, scheduled_terminate_at__lt=timezone.now() # subtract 10 minutes to give normal recycle task time to execute; # this housekeeping task is a final safety net if recycle fails - timedelta(minutes=10), ) except DBLabInstance.DoesNotExist: continue match stack_status: case "DELETE_IN_PROGRESS": # Already being deleted; skip and let it finish continue case "CREATE_COMPLETE" | "CREATE_FAILED": logger.info( f"Housekeeping[{region}]: Deleting stack {stack_name} in status {stack_status}" ) try: client.delete_stack(StackName=stack_name) except Exception as exc: logger.warning( f"Housekeeping[{region}]: Failed to delete {stack_name} in {stack_status}: {exc}" ) case "DELETE_FAILED": logger.info( f"Housekeeping[{region}]: DELETE_FAILED for {stack_name}, attempting to force delete" ) try: client.delete_stack( StackName=stack_name, DeletionMode="FORCE_DELETE_STACK" ) except Exception as exc: logger.error( f"Housekeeping[{region}]: Still failed to delete {stack_name}: {exc}" ) case _: continue