""" Query executor for filtering test tasks based on QueryFilter conditions. Translates QueryFilter objects into database queries. """ from typing import List from sqlalchemy import and_, not_, or_ from sqlalchemy.orm import Query from src.database.db import get_db from src.models.models import TestTask from src.services.workload_templates import ( iter_equivalent_templates, resolve_workload_template_from_filter_value, ) from .query_filter import FilterCondition, FilterDimension, FilterOperator, QueryFilter def query_tasks_by_filter(query_filter: QueryFilter, db_session=None) -> List[TestTask]: """Execute query to find tasks matching the filter conditions. Args: query_filter: QueryFilter with conditions to apply db_session: Optional database session. If provided, uses this session and returns objects bound to it. If None, creates a new session and preloads attributes. Returns: List of TestTask objects matching all conditions Example: >>> filter = QueryFilter() \\ ... .add_condition(FilterDimension.CLOUD_PROVIDER, value="AWS") \\ ... .add_condition(FilterDimension.MACHINE_GENERATION, value="G2") >>> tasks = query_tasks_by_filter(filter) """ if db_session: # Use provided session return _execute_query(db_session, query_filter) else: # Create new session with attribute preloading with get_db() as db: return _execute_query_with_preload(db, query_filter) def _execute_query(db, query_filter: QueryFilter) -> List[TestTask]: """Execute query without preloading (for use with active session).""" query = db.query(TestTask) # Apply task status filter if specified, otherwise default to SUCCESS + FAILURE status_condition = query_filter.get_condition(FilterDimension.TASK_STATUS) if status_condition: if status_condition.operator == FilterOperator.IN: query = query.filter(TestTask.status.in_(status_condition.value)) else: query = query.filter(TestTask.status == status_condition.value) else: # Default: include both SUCCESS and FAILURE tasks query = query.filter(TestTask.status.in_(["SUCCESS", "FAILURE"])) # Always filter out deleted tasks query = query.filter(TestTask.deleted == False) # Apply each filter condition for condition in query_filter.conditions: if condition.dimension != FilterDimension.TASK_STATUS: query = _apply_condition(query, condition) tasks = query.all() # Apply post-processing filters for computed fields tasks = post_filter_tasks(tasks, query_filter) return tasks def _execute_query_with_preload(db, query_filter: QueryFilter) -> List[TestTask]: """Execute query with attribute preloading (for detached use).""" query = db.query(TestTask) # Apply task status filter if specified, otherwise default to SUCCESS + FAILURE status_condition = query_filter.get_condition(FilterDimension.TASK_STATUS) if status_condition: if status_condition.operator == FilterOperator.IN: query = query.filter(TestTask.status.in_(status_condition.value)) else: query = query.filter(TestTask.status == status_condition.value) else: # Default: include both SUCCESS and FAILURE tasks query = query.filter(TestTask.status.in_(["SUCCESS", "FAILURE"])) # Always filter out deleted tasks query = query.filter(TestTask.deleted == False) # Apply each filter condition for condition in query_filter.conditions: if condition.dimension != FilterDimension.TASK_STATUS: query = _apply_condition(query, condition) # Execute query and load all attributes before session closes tasks = query.all() # Force load all needed attributes while session is still active for task in tasks: # Access attributes to trigger loading _ = task.id _ = task.cloud_provider _ = task.region _ = task.cluster_version _ = task.tidb_spu _ = task.tidb_count _ = task.tidb_config _ = task.tikv_spu _ = task.tikv_count _ = task.tikv_storage_type _ = task.tikv_storage_size_gi _ = task.tikv_config _ = task.pd_spu _ = task.pd_count _ = task.pd_config _ = task.tiflash_spu _ = task.tiflash_count _ = task.tiflash_storage_type _ = task.tiflash_storage_size_gi _ = task.tiflash_config _ = task.tiproxy_spu _ = task.tiproxy_count _ = task.tiproxy_config _ = task.workload_template _ = task.workflow_execution_id _ = task.workload_config _ = task.metadata _ = task.status # Apply post-processing filters for computed fields tasks = post_filter_tasks(tasks, query_filter) return tasks def count_matching_tasks(query_filter: QueryFilter) -> int: """Count tasks matching the filter without loading full objects. Args: query_filter: QueryFilter with conditions to apply Returns: Number of matching tasks """ with get_db() as db: query = db.query(TestTask) # Apply task status filter if specified, otherwise default to SUCCESS + FAILURE status_condition = query_filter.get_condition(FilterDimension.TASK_STATUS) if status_condition: if status_condition.operator == FilterOperator.IN: query = query.filter(TestTask.status.in_(status_condition.value)) else: query = query.filter(TestTask.status == status_condition.value) else: # Default: include both SUCCESS and FAILURE tasks query = query.filter(TestTask.status.in_(["SUCCESS", "FAILURE"])) # Always filter out deleted tasks query = query.filter(TestTask.deleted == False) for condition in query_filter.conditions: if condition.dimension != FilterDimension.TASK_STATUS: query = _apply_condition(query, condition) return query.count() def _apply_condition(query: Query, condition: FilterCondition) -> Query: """Apply a single filter condition to the query. Args: query: SQLAlchemy query object condition: Filter condition to apply Returns: Modified query with condition applied """ dimension = condition.dimension operator = condition.operator value = condition.value # Direct database fields if dimension == FilterDimension.CLOUD_PROVIDER: return _apply_simple_filter(query, TestTask.cloud_provider, operator, value) elif dimension == FilterDimension.REGION: return _apply_simple_filter(query, TestTask.region, operator, value) elif dimension == FilterDimension.CLUSTER_VERSION: return _apply_simple_filter(query, TestTask.cluster_version, operator, value) elif dimension == FilterDimension.TIDB_SPU: return _apply_simple_filter(query, TestTask.tidb_spu, operator, value) elif dimension == FilterDimension.TIKV_SPU: return _apply_simple_filter(query, TestTask.tikv_spu, operator, value) elif dimension == FilterDimension.TIDB_COUNT: return _apply_simple_filter(query, TestTask.tidb_count, operator, value) elif dimension == FilterDimension.TIKV_COUNT: return _apply_simple_filter(query, TestTask.tikv_count, operator, value) elif dimension == FilterDimension.TIFLASH_SPU: return _apply_simple_filter(query, TestTask.tiflash_spu, operator, value) elif dimension == FilterDimension.TIFLASH_COUNT: return _apply_simple_filter(query, TestTask.tiflash_count, operator, value) elif dimension == FilterDimension.TASK_STATUS: # Task status is handled separately in _execute_query return query # Computed/derived fields require post-filtering or special handling elif dimension == FilterDimension.MACHINE_GENERATION: return _apply_machine_generation_filter(query, operator, value) elif dimension == FilterDimension.TIPROXY_ENABLED: if value: query = query.filter(TestTask.tiproxy_spu.isnot(None)) query = query.filter(TestTask.tiproxy_count > 0) else: # Either tiproxy_spu is None or count is 0/None query = query.filter( (TestTask.tiproxy_spu.is_(None)) | (TestTask.tiproxy_count == 0) ) return query elif dimension == FilterDimension.WORKLOAD_TYPE: if operator == FilterOperator.EQUALS: if isinstance(value, str): resolved = resolve_workload_template_from_filter_value(value) if resolved: equivalents = list(iter_equivalent_templates(resolved)) return query.filter(TestTask.workload_template.in_(equivalents)) query = query.filter(TestTask.workload_template.like(f"%{value}%")) elif operator == FilterOperator.IN: conditions = [] for v in value: resolved = resolve_workload_template_from_filter_value(v) if resolved: equivalents = list(iter_equivalent_templates(resolved)) conditions.append(TestTask.workload_template.in_(equivalents)) else: conditions.append(TestTask.workload_template.like(f"%{v}%")) query = query.filter(or_(*conditions)) return query # If dimension not recognized, return query unchanged return query def _apply_simple_filter(query: Query, field, operator: FilterOperator, value) -> Query: """Apply a simple filter on a database field. Args: query: SQLAlchemy query field: Database field (e.g., TestTask.cloud_provider) operator: Filter operator value: Value to compare Returns: Modified query """ if operator == FilterOperator.EQUALS: return query.filter(field == value) elif operator == FilterOperator.NOT_EQUALS: return query.filter(field != value) elif operator == FilterOperator.IN: return query.filter(field.in_(value)) elif operator == FilterOperator.NOT_IN: return query.filter(~field.in_(value)) elif operator == FilterOperator.GREATER_THAN: return query.filter(field > value) elif operator == FilterOperator.GREATER_THAN_OR_EQUAL: return query.filter(field >= value) elif operator == FilterOperator.LESS_THAN: return query.filter(field < value) elif operator == FilterOperator.LESS_THAN_OR_EQUAL: return query.filter(field <= value) elif operator == FilterOperator.LIKE: return query.filter(field.like(value)) elif operator == FilterOperator.CONTAINS: return query.filter(field.contains(value)) return query def _apply_machine_generation_filter( query: Query, operator: FilterOperator, value ) -> Query: """Apply machine generation filter. NOTE: SPU naming is counterintuitive in the database: - G4 (Graviton4) → SPUs with "-V2" suffix in DB - G2 (Graviton2) → SPUs without version suffix in DB Args: query: SQLAlchemy query operator: Filter operator (EQUALS or IN) value: Generation string like "G2" or "G4", or list like ["G2", "G4"] Returns: Modified query """ def get_generation_conditions(gen: str): """Get SQL conditions for a generation value.""" if gen == "G4": # G4 (Graviton4) → SPUs with -V2 suffix return [ TestTask.tidb_spu.like("%-V2"), TestTask.tikv_spu.like("%-V2"), ] elif gen == "G2": # G2 (Graviton2) → SPUs without version suffix # Match SPUs that don't have -V in them return [ and_( TestTask.tidb_spu.isnot(None), not_(TestTask.tidb_spu.like("%-V%")), ), and_( TestTask.tikv_spu.isnot(None), not_(TestTask.tikv_spu.like("%-V%")), ), ] else: # Unknown generation return [] if operator == FilterOperator.EQUALS: conditions = get_generation_conditions(value) if conditions: query = query.filter(or_(*conditions)) elif operator == FilterOperator.IN: all_conditions = [] for gen in value: all_conditions.extend(get_generation_conditions(gen)) if all_conditions: query = query.filter(or_(*all_conditions)) return query def _apply_computed_cores_filter( query: Query, component: str, operator: FilterOperator, value: int ) -> Query: """Apply filter on computed total cores. Since total cores = cores_per_node × count, and cores_per_node is embedded in the SPU string, we need to filter in Python after loading tasks. For efficiency, we return all tasks and filter will be applied later. Note: This is a limitation of the current approach. For better performance, we could add computed columns to the database. Args: query: SQLAlchemy query component: "tidb" or "tikv" operator: Filter operator value: Target total cores Returns: Query (filtering will be done in post-processing) """ # For now, return query as-is and filter in post-processing # TODO: Implement post-processing filter or add computed columns to DB return query def _apply_computed_memory_filter( query: Query, component: str, operator: FilterOperator, value: int ) -> Query: """Apply filter on computed total memory. Similar to cores filter, this requires post-processing. Args: query: SQLAlchemy query component: "tidb" or "tikv" operator: Filter operator value: Target total memory in GB Returns: Query (filtering will be done in post-processing) """ # For now, return query as-is and filter in post-processing # TODO: Implement post-processing filter return query def post_filter_tasks( tasks: List[TestTask], query_filter: QueryFilter ) -> List[TestTask]: """Apply post-processing filters for computed fields. Most filters are now applied directly at SQL level. This function is kept for future extensibility. Args: tasks: List of tasks from database query query_filter: Original query filter Returns: Filtered list of tasks (currently unmodified) """ # All current filters can be applied at SQL level # This function is kept for future computed field filters return tasks def _compare(actual_value, operator: FilterOperator, target_value) -> bool: """Compare values using the given operator. Args: actual_value: Actual value from task operator: Comparison operator target_value: Target value from filter Returns: True if comparison succeeds """ if operator == FilterOperator.EQUALS: return actual_value == target_value elif operator == FilterOperator.NOT_EQUALS: return actual_value != target_value elif operator == FilterOperator.IN: return actual_value in target_value elif operator == FilterOperator.NOT_IN: return actual_value not in target_value elif operator == FilterOperator.GREATER_THAN: return actual_value > target_value elif operator == FilterOperator.GREATER_THAN_OR_EQUAL: return actual_value >= target_value elif operator == FilterOperator.LESS_THAN: return actual_value < target_value elif operator == FilterOperator.LESS_THAN_OR_EQUAL: return actual_value <= target_value return True if __name__ == "__main__": # Test the query executor print("Testing Query Executor:\n") # Test 1: Simple filter filter1 = ( QueryFilter() .add_condition(FilterDimension.CLOUD_PROVIDER, value="AWS") .add_condition(FilterDimension.REGION, value="us-west-2") ) print("Test 1: Simple filter (Cloud + Region)") print(f" Filter: {filter1.to_description()}") tasks1 = query_tasks_by_filter(filter1) count1 = count_matching_tasks(filter1) print(f" Found {len(tasks1)} tasks (count method: {count1})") if tasks1: print(f" Sample task: ID={tasks1[0].id}, TiDB={tasks1[0].tidb_spu}") print() # Test 2: Machine generation filter (G2 - Graviton2) filter2 = ( QueryFilter() .add_condition(FilterDimension.CLOUD_PROVIDER, value="AWS") .add_condition(FilterDimension.MACHINE_GENERATION, value="G2") ) print("Test 2: Machine generation filter (G2 - Graviton2)") print(f" Filter: {filter2.to_description()}") count2 = count_matching_tasks(filter2) print(f" Found {count2} tasks (without -V suffix)") print() # Test 2b: Machine generation filter (G4 - Graviton4) filter2b = ( QueryFilter() .add_condition(FilterDimension.CLOUD_PROVIDER, value="AWS") .add_condition(FilterDimension.MACHINE_GENERATION, value="G4") ) print("Test 2b: Machine generation filter (G4 - Graviton4)") print(f" Filter: {filter2b.to_description()}") count2b = count_matching_tasks(filter2b) print(f" Found {count2b} tasks (with -V2 suffix)") print() # Test 3: Computed cores filter (with post-processing) filter3 = ( QueryFilter() .add_condition(FilterDimension.CLOUD_PROVIDER, value="AWS") .add_condition( FilterDimension.TIDB_TOTAL_CORES, operator=FilterOperator.GREATER_THAN_OR_EQUAL, value=64, ) ) print("Test 3: Computed cores filter (auto post-processing)") print(f" Filter: {filter3.to_description()}") tasks3 = query_tasks_by_filter(filter3) # Now includes post-filtering print(f" Found {len(tasks3)} tasks") print("\n✓ Query executor tests completed!") print(" - Simple filters working") print(" - Machine generation filter working") print(" - Computed field filtering working")