import logging from datetime import timedelta from alibabacloud_tea_openapi.models import Config as AlicloudConfig from alibabacloud_ros20190910.client import Client as AlicloudROSClient from alibabacloud_ros20190910.models import ( DeleteStackRequest, ListStacksRequest, ) from celery import shared_task from django.conf import settings from django.utils import timezone from labs.models import LabInstance as DBLabInstance logger = logging.getLogger(__name__) ALICLOUD_REGIONS = [ "ap-southeast-1", ] @shared_task def alicloud_ros_housekeeping(): """ Fallback cleanup task for Alibaba Cloud ROS stacks. Purpose: - Acts as a safety net when a Lab Instance is not properly destroyed. - Scans configured regions and removes orphaned/expired stacks tied to lab instances. - Reduces cost by preventing unused resources from lingering. """ for region in ALICLOUD_REGIONS: _alicloud_ros_housekeeping(region) def _alicloud_ros_housekeeping(region: str): client = AlicloudROSClient( config=AlicloudConfig( access_key_id=settings.ALICLOUD_ROS_ACCESS_KEY_ID, access_key_secret=settings.ALICLOUD_ROS_ACCESS_KEY_SECRET, region_id=region, ) ) page_size = 50 page_number = 1 while True: response = client.list_stacks( request=ListStacksRequest( region_id=region, page_size=page_size, page_number=page_number, ) ) stacks = response.body.stacks for stack in stacks: stack_id = stack.stack_id stack_name = stack.stack_name stack_status = stack.status try: DBLabInstance.objects.get( unique_tag=stack_name, scheduled_terminate_at__lt=timezone.now() # subtract 10 minutes to give normal recycle task time to execute; # this housekeeping task is a final safety net if recycle fails - timedelta(minutes=10), ) except DBLabInstance.DoesNotExist: continue match stack_status: case "CREATE_COMPLETE" | "CREATE_FAILED": logger.info( f"Housekeeping[{region}]: Deleting stack {stack_name} in status {stack_status}" ) try: client.delete_stack( request=DeleteStackRequest( region_id=region, stack_id=stack_id, ) ) except Exception as e: logger.error( f"Housekeeping[{region}]: Failed to delete stack {stack_name}: {e}" ) case "DELETE_FAILED": logger.error( f"Housekeeping[{region}]: Find a stack in DELETE_FAILED status: {stack_name}, " "please check manually" ) case _: continue total_count = response.body.total_count if page_number * page_size >= total_count: break page_number += 1