"""Test concurrency control for nested insight service (database-driven architecture).""" import asyncio from datetime import datetime, timedelta, timezone from unittest.mock import MagicMock import pytest from aiops.infra.config.services import NestedInsightSetting, ServicesSetting from aiops.models.guardian.nested_insight import NestedInsightStatus from aiops.service.guardian.nested_insight.service import ( NestedInsightService, WindowSnapshot, ) @pytest.fixture def concurrency_config(): """Configuration with limited concurrency for testing.""" return NestedInsightSetting( enabled=True, max_concurrent_analyses=2, max_retry_attempts=3, ) @pytest.fixture def mock_service(concurrency_config): """Create a mocked service instance.""" logger = MagicMock() db = MagicMock() sched = MagicMock() asset = MagicMock() service_llm = MagicMock() services_config = ServicesSetting() service = NestedInsightService( logger=logger, db=db, sched=sched, asset=asset, service_llm=service_llm, config=concurrency_config, analysis_storage_dir=services_config.analysis_storage_dir, ) return service @pytest.mark.asyncio async def test_semaphore_initialization(mock_service: NestedInsightService): """Test that semaphore is initialized with correct limit.""" # Semaphore should be created in __init__ assert mock_service._semaphore is not None assert ( mock_service._semaphore._value == mock_service._config.max_concurrent_analyses ) # Active analyses set should be empty initially assert len(mock_service._active_analyses) == 0 @pytest.mark.asyncio async def test_semaphore_concurrency_control(mock_service: NestedInsightService): """Test that semaphore limits concurrent execution.""" execution_count = 0 max_concurrent = 0 current_concurrent = 0 lock = asyncio.Lock() async def mock_analysis(window, *args, **kwargs): nonlocal execution_count, max_concurrent, current_concurrent async with lock: current_concurrent += 1 max_concurrent = max(max_concurrent, current_concurrent) await asyncio.sleep(0.1) # Simulate work async with lock: current_concurrent -= 1 execution_count += 1 # Replace the analysis method mock_service._analyze_window_background = mock_analysis # Launch multiple tasks directly num_tasks = 5 tasks = [] for i in range(num_tasks): window = WindowSnapshot( id=i, tenant_id="test", cluster_id="cluster1", start_at=datetime.now(timezone.utc), end_at=datetime.now(timezone.utc) + timedelta(minutes=10), status=NestedInsightStatus.WATCHING, final_report_s3_key=None, ) mock_service._active_analyses.add(window.id) task = asyncio.create_task( mock_service._analyze_window_with_semaphore(window, [], None) ) tasks.append(task) # Wait for all tasks to complete await asyncio.gather(*tasks, return_exceptions=True) # All tasks should have been executed assert execution_count == num_tasks # Concurrent execution should not exceed limit assert max_concurrent <= mock_service._config.max_concurrent_analyses # Active analyses should be empty after completion assert len(mock_service._active_analyses) == 0 @pytest.mark.asyncio async def test_active_analyses_tracking(mock_service: NestedInsightService): """Test that _active_analyses set correctly tracks running tasks.""" async def mock_analysis(window, *args, **kwargs): # Check that window is in active set during execution assert window.id in mock_service._active_analyses await asyncio.sleep(0.05) mock_service._analyze_window_background = mock_analysis window = WindowSnapshot( id=1, tenant_id="test", cluster_id="cluster1", start_at=datetime.now(timezone.utc), end_at=datetime.now(timezone.utc) + timedelta(minutes=10), status=NestedInsightStatus.WATCHING, final_report_s3_key=None, ) # Add to active set and launch task mock_service._active_analyses.add(window.id) task = asyncio.create_task( mock_service._analyze_window_with_semaphore(window, [], None) ) # Should be in active set while running await asyncio.sleep(0.01) assert 1 in mock_service._active_analyses # Wait for completion await task # Should be removed from active set after completion assert 1 not in mock_service._active_analyses @pytest.mark.asyncio async def test_task_cleanup_on_error(mock_service: NestedInsightService): """Test that tasks are cleaned from _active_analyses even on error.""" async def mock_analysis_with_error(window, *args, **kwargs): raise Exception("Simulated error") mock_service._analyze_window_background = mock_analysis_with_error window = WindowSnapshot( id=1, tenant_id="test", cluster_id="cluster1", start_at=datetime.now(timezone.utc), end_at=datetime.now(timezone.utc) + timedelta(minutes=10), status=NestedInsightStatus.WATCHING, final_report_s3_key=None, ) # Add to active set and launch task mock_service._active_analyses.add(window.id) task = asyncio.create_task( mock_service._analyze_window_with_semaphore(window, [], None) ) # Wait for task to fail await task # Should be removed from active set even after error assert 1 not in mock_service._active_analyses @pytest.mark.asyncio async def test_shutdown_with_active_tasks(mock_service: NestedInsightService): """Test that shutdown logs active tasks but doesn't wait (stateless design).""" # Simulate some active analyses mock_service._active_analyses.add(1) mock_service._active_analyses.add(2) # Shutdown should complete immediately without waiting await mock_service.shutdown() # Shutdown event should be set assert mock_service._shutdown_event.is_set() # Active analyses are still tracked (database will handle recovery) assert len(mock_service._active_analyses) == 2