import datetime import logging import os import re import sys import typing from urllib.parse import urlparse from grpc_requests import Client import grpc import requests MGMT_SVC_ADDR = os.getenv('MGMT_SVC_ADDR', 'tidb-mgmt-service.tidb-management-service:10001') KEEP_HOUR = int(os.getenv('KEEP_HOUR', '12')) RETENTION_HOUR_REGEX = re.compile(r'keep-(\d+)h', re.IGNORECASE) FEISHU_WEBHOOK_URL = "https://open.feishu.cn/open-apis/bot/v2/hook/bd35d09e-2074-4811-8422-ba21b6b948ac" FSN_RESOURCE_POOL_SERVICE = 'tidb_mgmt_service.v1.ResourcePoolService' MN_LIST_RESOURCE_POOLS = 'ListResourcePools' MN_DELETE_RESOURCE_POOL = 'DeleteResourcePool' CUSTOM_METADATA = {} logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') def _get_resource_pool_client(addr: str) -> Client: """Initializes and returns a grpc-requests Client.""" logging.debug(f"Establishing gRPC client to {addr}") return Client.get_by_endpoint(addr) def _grpc_request(client: Client, service_name: str, method_name: str, request_data: dict, metadata: dict) -> typing.Any: """ Performs a dynamic gRPC request and handles common errors. Returns the response body (dict). """ metadata_list = [(k.lower(), str(v)) for k, v in metadata.items()] full_method_name = f"{service_name}/{method_name}" logging.debug(f"Request: {full_method_name}, Data: {request_data}, Metadata: {metadata_list}") try: response = client.request( service_name, method_name, request_data, metadata=metadata_list, timeout=15 ) logging.info(f"Response: {response}") return response except grpc.RpcError as e: error_details = f"Status Code: {e.code().name}, Details: {e.details()}" logging.error(f"gRPC call failed for {full_method_name}: {error_details}") raise ConnectionError(f"gRPC RPC Error: {error_details}") from e except Exception as e: logging.error(f"An unexpected error occurred during gRPC call to {full_method_name}: {e}") raise RuntimeError(f"Unexpected gRPC Error: {e}") from e def _get_retention_hour(resource_pool_name: str, default_hour: int) -> int: """ Extracts retention hour from resource pool name (e.g., 'keep-48h'). """ match = RETENTION_HOUR_REGEX.search(resource_pool_name) if match: try: retention_h = int(match.group(1)) logging.debug(f"Found specific retention hour: {retention_h} for {resource_pool_name}") return retention_h except ValueError: logging.warning(f"Could not parse retention hour from '{match.group(1)}' in '{resource_pool_name}'. Using default.") return default_hour return default_hour def _is_expired(create_time_str: str, retention_hour: int) -> bool: """ Checks if a resource pool has expired. """ try: create_time = datetime.datetime.fromisoformat(create_time_str.replace('Z', '+00:00')) if create_time.tzinfo is None: create_time = create_time.replace(tzinfo=datetime.timezone.utc) now = datetime.datetime.now(datetime.timezone.utc) retention_delta = datetime.timedelta(hours=retention_hour) expire_time = create_time + retention_delta return now > expire_time except ValueError as e: logging.error(f"Error parsing creation time '{create_time_str}': {e}. Cannot determine expiration.") return False def process_resource_pool_cleanup(dry_run: bool = False): """ Lists all resource pools, checks for expiration and usage, and deletes expired, unused ones. """ client = _get_resource_pool_client(MGMT_SVC_ADDR) # List to store deletion failures deletion_failures = [] action_type = "DRY-RUN" if dry_run else "DELETION" logging.info(f"--- Starting Resource Pool Cleanup ({action_type}) ---") logging.info(f"Default Retention Hour: {KEEP_HOUR}h.") # 1. Get all resource pools try: logging.info("Calling ListResourcePools to fetch all resource pools...") list_response = _grpc_request( client, FSN_RESOURCE_POOL_SERVICE, MN_LIST_RESOURCE_POOLS, { "page_size": 50, "view": "DETAIL", }, CUSTOM_METADATA ) resource_pools_profiles = list_response.get('resource_pools', []) total_pools = len(resource_pools_profiles) logging.info(f"Successfully retrieved {total_pools} resource pool profiles.") except Exception as e: logging.critical("Failed to list resource pools. Cannot proceed with cleanup.") # Attempt to close client, then re-raise the exception try: client.close() except AttributeError: logging.warning("Client object does not have a 'close' method. Ignoring closure.") # Re-raise the exception to terminate the script raise RuntimeError("Cleanup process terminated due to failure in listing resource pools.") from e pools_to_delete = [] # ... (Step 2: Iterate and determine which pools to delete - this part remains the same) ... # The code from line 124 to 157 in the original file should be here. for rp_profile in resource_pools_profiles: rp = rp_profile.get('resource_pool', {}) instances = rp_profile.get('instances', []) rp_id = str(rp.get('id', 'N/A')) rp_name = rp.get('name', 'N/A') rp_create_time = rp.get('create_time', None) log_prefix = f"[RP ID: {rp_id}, Name: {rp_name}]" if instances: instance_count = len(instances) logging.info(f"{log_prefix} Skipping. In use by {instance_count} instance(s).") continue if not rp_create_time: logging.warning(f"{log_prefix} Skipping. Missing createTime information.") continue retention_h = _get_retention_hour(rp_name, KEEP_HOUR) if not _is_expired(rp_create_time, retention_h): logging.info(f"{log_prefix} Skipping. Created at {rp_create_time} (Keep: {retention_h}h). Not yet expired.") continue logging.warning(f"{log_prefix} ELIGIBLE for DELETION. Unused and expired (Keep: {retention_h}h, Created: {rp_create_time}).") pools_to_delete.append(rp_id) # ... (End of Step 2) ... logging.info(f"Found {len(pools_to_delete)} resource pools eligible for deletion.") # 3. Delete the eligible resource pools with stability if not pools_to_delete: logging.info("No resource pools to delete. Cleanup finished.") try: client.close() except AttributeError: logging.warning("Client object does not have a 'close' method. Ignoring closure.") return deleted_count = 0 for rp_id in pools_to_delete: delete_request = {"resource_pool_id": rp_id} log_prefix = f"[RP ID: {rp_id}]" if dry_run: logging.info(f"{log_prefix} [DRY-RUN] Would have called DeleteResourcePool with data: {delete_request}") deleted_count += 1 continue try: logging.info(f"{log_prefix} Attempting actual deletion...") _grpc_request( client, FSN_RESOURCE_POOL_SERVICE, MN_DELETE_RESOURCE_POOL, delete_request, CUSTOM_METADATA ) logging.info(f"{log_prefix} Successfully deleted.") deleted_count += 1 except Exception as e: error_msg = f"Deletion failed for RP ID {rp_id}. Error: {e}" logging.error(f"{log_prefix} {error_msg}") # Record the failure for later summary deletion_failures.append(error_msg) # Continue to the next pool continue success_count = deleted_count - len(deletion_failures) # Count of successful deletions logging.info(f"Cleanup finished. Successfully deleted {success_count} pools out of {len(pools_to_delete)} eligible.") try: client.close() except AttributeError: logging.warning("Client object does not have a 'close' method. Ignoring closure.") if deletion_failures: failure_summary = "\n".join(deletion_failures) failure_count = len(deletion_failures) logging.error(f"\n--- Deletion Failures Summary ({len(deletion_failures)} total) ---\n{failure_summary}\n--------------------------------------") send_feishu_simple_notification(f"resource pool clean failed: {failure_count} deletion(s) failed. Check CI logs for details.") raise RuntimeError(f"Cleanup finished with {len(deletion_failures)} resource pool deletion failures. See logs for details.") def send_feishu_simple_notification(message: str): """ Sends a simple text message to the Feishu webhook. """ if not FEISHU_WEBHOOK_URL: logging.warning("FEISHU_WEBHOOK_URL is not set. Skipping Feishu notification.") return payload = { "msg_type": "text", "content": { "text": message } } try: logging.info(f"Attempting to send simple Feishu notification...") response = requests.post(FEISHU_WEBHOOK_URL, json=payload, timeout=5) response.raise_for_status() logging.info("Simple Feishu notification sent successfully.") except requests.exceptions.RequestException as e: logging.error(f"Failed to send simple Feishu notification: {e}") def process(): """Main function to start the cleanup process, checking for dry-run flag.""" is_dry_run = os.getenv('DRY_RUN', 'False').lower() in ('true', '1', 't') process_resource_pool_cleanup(dry_run=is_dry_run) if __name__ == '__main__': process()