import difflib import re def normalize_question(question): if question is None: return None # Trim whitespace and remove all trailing punctuation # Remove trailing punctuation (one or more of '.', ',', '!', '?') question = re.sub(r"[.,!?]+$", "", question.strip()) # Convert to lowercase return question.lower() def extract_question(qa_content): # Use regular expression to find the text after "User question:" match = re.search(r"User question:\s*(.*)", qa_content, re.IGNORECASE) if match: # Extract the matched question and process it question = match.group(1).strip() # Trim leading and trailing whitespace return normalize_question(question) return None class SimpleCache: def __init__(self, items: list = []): self.cache = [] self.questions_in_cache = [] for item in items: self.add(item["question"], item["job_id"]) def add(self, question, job_id): """ Adds a question and job_id to the cache. """ question = normalize_question(question) if question is None or question in self.questions_in_cache: return self.cache.append({"question": normalize_question(question), "job_id": job_id}) self.questions_in_cache.append(normalize_question(question)) def get(self, question): """ Retrieves the job_id for a given question using simple string matching. """ question = normalize_question(question) if question is None: return None closest_match = difflib.get_close_matches( question, self.questions_in_cache, n=1, cutoff=0.90 ) if closest_match: for item in self.cache: if item["question"] == closest_match[0]: return item["job_id"] return None # It's a workaround solution to reduce duplicated job validation common_questions = [ { "question": "Does TiDB support serverless?", "job_id": "b936c233-5731-43cc-b483-281623d648a5", }, { "question": "Does TiDB support vector?", "job_id": "98c50963-7485-475e-9dc9-45568d9dafce", }, { "question": "Does TiDB support vector search?", "job_id": "98c50963-7485-475e-9dc9-45568d9dafce", }, { "question": "Does TiDB support FOREIGN KEY?", "job_id": "b0567683-257e-4006-ab34-ef3238fcc538", }, {"question": "What is TiDB?", "job_id": "5185084f-6dca-4afd-8df5-cd98cf975098"}, {"question": "what's tidb?", "job_id": "5185084f-6dca-4afd-8df5-cd98cf975098"}, ] simple_semantic_cache = SimpleCache(common_questions)