import asyncio from queue import Queue from threading import Thread import uuid from typing import Callable, Any import logging import traceback from app.settings import settings logger = logging.getLogger(__name__) class TaskQueue: def __init__(self, max_concurrent_tasks: int): self.task_queue = Queue() self.max_concurrent_tasks = max_concurrent_tasks self.semaphore = asyncio.Semaphore(max_concurrent_tasks) def start_workers(self, worker_func: Callable): for _ in range(self.max_concurrent_tasks): Thread(target=self._worker_thread, args=(worker_func,), daemon=True).start() def _worker_thread(self, worker_func: Callable): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) while True: job_id, job_request = self.task_queue.get() try: asyncio.run(self._run_task(worker_func, job_id, job_request)) except Exception as e: error_stack = traceback.format_exc() logger.error(f"Error processing job {job_id}: {e}\n{error_stack}") finally: self.task_queue.task_done() async def _run_task( self, worker_func: Callable, job_id: uuid.UUID, job_request: Any ): async with self.semaphore: await worker_func(job_id, job_request) def add_task(self, job_id: uuid.UUID, job_request: Any): self.task_queue.put((job_id, job_request)) # create global task queue instance task_queue = TaskQueue(max_concurrent_tasks=settings.TASK_QUEUE_WORKERS)