# from __future__ import annotations import hashlib import json import logging import sqlite3 from copy import deepcopy from dataclasses import dataclass, fields from enum import Enum from typing import Any, Dict, List, Optional, Tuple import dacite from environs import Env from database.dao import OpenAPITestCaseCollectionDAO, OpenAPITestCaseGenerationDAO from utils.sqlite2yaml import convert_sqlite_to_yaml env = Env() env.read_env(".env") HOST = env.str("HOST") ORG_ID = env.str("ORG_ID") PRJ_ID = env.str("PRJ_ID") AUTH = { "public_key": env.str("PUBLIC_KEY"), "private_key": env.str("PRIVATE_KEY"), } def json_dumps_or_none(value: Any) -> Optional[str]: if value is None: return None return json.dumps(value) @dataclass class Case: operation_id: str method: Optional[str] = None path: Optional[str] = None category: Optional[str] = None affected_element: Optional[str] = None element_type: Optional[str] = None generation_strategy: Optional[dict] = None type: str = "Negative" description: Optional[str] = None environment: Optional[dict] = None request: Optional[dict] = None expected_status: Optional[int] = None confirmed: bool = False ignored: bool = False def __post_init__(self): if self.environment is None: self.environment = {} for var in ["host", "org_id", "prj_id"]: if self.environment.get(var) is None: value = env.str(var.upper()) if value is None: raise ValueError(f"{var.upper()} is not set") self.environment[var] = value if "auth" not in self.environment or not isinstance( self.environment["auth"], dict ): if env.str("PUBLIC_KEY") is None or env.str("PRIVATE_KEY") is None: raise ValueError("PUBLIC_KEY or PRIVATE_KEY is not set") self.environment["auth"] = { "public_key": env.str("PUBLIC_KEY"), "private_key": env.str("PRIVATE_KEY"), } @property def id(self) -> str: if self.type == "Negative": if ( self.generation_strategy is None or self.affected_element is None or self.category is None ): raise ValueError( "Case ID is generated from category[str], affected_element[str] and generation_strategy[Dict[str, str]]" ) id_string = f"{self.operation_id}_{self.category}_{self.affected_element}_{self.generation_strategy['name']}_{self.generation_strategy['value']}" # type: ignore else: id_string = f"{self.operation_id}_{self.description}" # type: ignore return hashlib.md5(id_string.encode()).hexdigest() @classmethod def from_dict(cls, data: Dict[str, Any]) -> "Case": path_params_in_path = [ item[1:-1] for item in data["path"].split(":")[0].split("/") if item.startswith("{") and item.endswith("}") ] if len(path_params_in_path) > 0 and data.get("expected_status"): if data["request"].get("path_params"): existing_path_params = set(data["request"]["path_params"].keys()) if existing_path_params - set(path_params_in_path): logging.warning( f"Path parameters in path are not in request path_params: {existing_path_params - set(path_params_in_path)}" ) else: for path_param in path_params_in_path: if path_param not in data["request"]["path_params"]: data["request"]["path_params"][path_param] = "placeholder" else: data["request"]["path_params"] = {} for path_param in path_params_in_path: data["request"]["path_params"][path_param] = "placeholder" config = dacite.Config( type_hooks={ Dict: dict, }, ) return dacite.from_dict(data_class=cls, data=data, config=config) def compare(self, other: "Case") -> Dict[str, tuple]: diff = {} for field in fields(self): self_value = getattr(self, field.name) other_value = getattr(other, field.name) if isinstance(self_value, dict): self_value = json.dumps(self_value, sort_keys=True) if isinstance(other_value, dict): other_value = json.dumps(other_value, sort_keys=True) if self_value != other_value: diff[field.name] = (self_value, other_value) return diff def __eq__(self, other) -> bool: if not isinstance(other, Case): return NotImplemented return self.compare(other) == {} class CaseManager: def __init__(self, db_path: str): self.conn = sqlite3.connect(db_path) self.create_table() self.metadata: Optional[Dict[str, Any]] = None def create_table(self) -> None: cursor = self.conn.cursor() cursor.execute( """ CREATE TABLE IF NOT EXISTS cases ( id TEXT PRIMARY KEY, operation_id TEXT, method TEXT DEFAULT NULL, path TEXT DEFAULT NULL, environment JSON DEFAULT NULL, request JSON DEFAULT NULL, type TEXT, category TEXT DEFAULT NULL, description TEXT DEFAULT NULL, affected_element TEXT DEFAULT NULL, element_type TEXT DEFAULT NULL, generation_strategy JSON DEFAULT NULL, expected_status INTEGER DEFAULT NULL, confirmed BOOLEAN DEFAULT FALSE, ignored BOOLEAN DEFAULT FALSE ) """ ) self.conn.commit() def get_case(self, case_id: str) -> Optional[Case]: cursor = self.conn.cursor() cursor.execute("SELECT * FROM cases WHERE id = ?", (case_id,)) row = cursor.fetchone() if row: return Case( method=row[2], path=row[3], operation_id=row[1], environment=json.loads(row[4]) if row[4] else None, request=json.loads(row[5]) if row[5] else None, type=row[6], category=row[7], description=row[8], affected_element=row[9], element_type=row[10], generation_strategy=json.loads(row[11]) if row[11] else None, expected_status=row[12], confirmed=row[13], ignored=row[14], ) return None def execute_query( self, query: str, params: Optional[Tuple[Any]] = None ) -> List[Case]: cursor = self.conn.cursor() if params: cursor.execute(query, params) else: cursor.execute(query) rows = cursor.fetchall() cases = [] for row in rows: cases.append( Case( method=row[2], path=row[3], operation_id=row[1], environment=json.loads(row[4]) if row[4] else None, request=json.loads(row[5]) if row[5] else None, type=row[6], category=row[7], description=row[8], affected_element=row[9], element_type=row[10], generation_strategy=json.loads(row[11]) if row[11] else None, expected_status=row[12], confirmed=row[13], ignored=row[14], ) ) return cases def add_or_update_case(self, case: Case) -> None: existing_case = self.get_case(case.id) if existing_case == case: return if existing_case: diff = existing_case.compare(case) if existing_case.confirmed: print(f"WARNING: Case {case.id} already exists and is confirmed.") print(f"WARNING: Diff fields: {diff}") self._update_case(case) else: print(f"INFO: Case {case.id} already exists but is not confirmed.") print(f"INFO: Diff fields: {diff}") self._update_case(case) else: self._add_case(case) self.conn.commit() def _add_case(self, case: Case) -> None: cursor = self.conn.cursor() cursor.execute( """ INSERT INTO cases (id, operation_id, method, path, environment, request, type, category, description, affected_element, element_type, generation_strategy, expected_status, confirmed, ignored) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( case.id, case.operation_id, case.method.upper() if case.method else None, case.path, json_dumps_or_none(case.environment), json_dumps_or_none(case.request), case.type, case.category, case.description, case.affected_element, case.element_type, json_dumps_or_none(case.generation_strategy), case.expected_status, case.confirmed, case.ignored, ), ) def _update_case(self, case: Case) -> None: cursor = self.conn.cursor() cursor.execute( """ UPDATE cases SET operation_id = ?, method = ?, path = ?, environment = ?, request = ?, type = ?, category = ?, description = ?, affected_element = ?, element_type = ?, generation_strategy = ?, expected_status = ?, confirmed = ?, ignored = ? WHERE id = ? """, ( case.operation_id, case.method.upper() if case.method else None, case.path, json_dumps_or_none(case.environment), json_dumps_or_none(case.request), case.type, case.category, case.description, case.affected_element, case.element_type, json_dumps_or_none(case.generation_strategy), case.expected_status, case.confirmed, case.ignored, case.id, ), ) def _get_diff(self, old_case: dict, new_case: dict) -> Optional[str]: diff = [] for key in old_case.keys(): new_value = ( json.dumps(new_case.get(key), sort_keys=True) if new_case.get(key) is not None else None ) if old_case[key] != new_value: if old_case[key] is not None: diff.append(f"{key}: {old_case[key]} -> {new_value}") return "\n".join(diff) if diff else None def delete_case(self, case_id: str) -> None: cursor = self.conn.cursor() cursor.execute("DELETE FROM cases WHERE id = ?", (case_id,)) self.conn.commit() def close(self) -> None: self.conn.close() def set_metadata(self, metadata: Dict[str, Any]) -> None: self.metadata = metadata def save_to_sqlite(self, target_path: str): """Copy in-memory database to a file.""" import sqlite3 # Create a new database file target_conn = sqlite3.connect(target_path) # Copy all data from memory to file with target_conn: self.conn.backup(target_conn) target_conn.close() def save_to_yaml(self, target_path: str): if not target_path.endswith(".yaml"): target_path = target_path.split(".")[0] + ".yaml" convert_sqlite_to_yaml(self, target_path, self.metadata) def save_to_tidb(self): metadata_dao = OpenAPITestCaseGenerationDAO() metadata_dao.insert(self.metadata) testcase_dao = OpenAPITestCaseCollectionDAO() rows = self.execute_query("SELECT * FROM cases") testcase_dao.batch_insert( rows, self.metadata["trace_id"], self.metadata["trace_id"] ) def get_cases_by_prefix(self, prefix: str, order_by: str, ascending: bool) -> List[Case]: rows = self.execute_query(f"SELECT * FROM cases WHERE expected_status IS NOT NULL AND operation_id LIKE '{prefix}%' ORDER BY {order_by} {'ASC' if ascending else 'DESC'}") return rows def convert_case_to_multistep_style(_case: Case) -> dict: # ONLY FOR AUDITLOG NOW case_in_multistep_style = {} case_in_multistep_style["case_id"] = _case.id case_in_multistep_style["operation_id"] = _case.operation_id case_in_multistep_style["description"] = _case.description case_in_multistep_style["case_type"] = _case.type case_in_multistep_style["request_type"] = "http" path_params = deepcopy(_case.request.get("path_params", {})) body = deepcopy(_case.request.get("body", {})) for key, value in path_params.items(): if value == "placeholder": if key.endswith("clusterId"): path_params[key] = "cluster_1" elif key.endswith("auditLogFilterRuleId"): path_params[key] = "auditlogfilterrule_1" else: print(f"WARNING: Unknown path parameter {key} in case {_case.id} with placeholder value") # nodeSpecKey, componentType, regionId, tidb.cloud/project, endpointId, networkContainerId # regionId_666, tidbCloudRegionId, tidbCloudRegionId_6666, customerAccountId, customerAccountId_666 # customerVpcCidr, customerVpcCidr_666, customerVpcId, customerVpcId_666 # maintenanceWindowId new_body, body_params = _convert_body(body) case_in_multistep_style["request"] = { "auth": { "auth_type": "digest", "content": _case.environment["auth"] }, "method": _case.method.upper(), "path": _case.path, "path_params": path_params, "query_params": _case.request.get("query_params", {}), "headers": {}, "body": new_body, "body_params": body_params } case_in_multistep_style["expect"] = {"status_code": _case.expected_status, "cel": ""} case_in_multistep_style["step_type"] = "generated" case_in_multistep_style["max_retries"] = 0 case_in_multistep_style["delay_after"] = 0 case_in_multistep_style["name"] = _case.operation_id case_in_multistep_style["save"] = None case_in_multistep_style["continue_on_unexpected"] = True case_in_multistep_style["meta"] = { "category": _case.category, "affected_element": _case.affected_element, "element_type": _case.element_type, "generation_strategy": _case.generation_strategy, "ignored": _case.ignored } case_in_multistep_style["stage"] = "execute" return case_in_multistep_style def _convert_body(body: dict) -> Tuple[dict, dict]: """ Recursively process the body dictionary to find all keys with value "placeholder". Replace "placeholder" with "{key}" in the body and add the key to body_params. Args: body: The input body dictionary to process Returns: A tuple containing (processed_body, body_params) """ placeholder_mapping = { "regionId": "region_id", "tidb.cloud/project": "project_id", } body_params = {} processed_body = deepcopy(body) def _process_item(item, path=None): if path is None: path = [] if isinstance(item, dict): for key, value in list(item.items()): if value == "placeholder": # Replace "placeholder" with "{key}" item[key] = f"{{{key}}}" # Add key to body_params body_params[key] = placeholder_mapping.get(key, "placeholder") # change later elif isinstance(value, (dict, list)): # Recursively process nested dictionaries and lists _process_item(value, path + [key]) elif isinstance(item, list): for i, element in enumerate(item): if isinstance(element, (dict, list)): # Recursively process nested dictionaries and lists in the list _process_item(element, path + [i]) _process_item(processed_body) return processed_body, body_params if __name__ == "__main__": # Test the convert_body function test_input = { "body": { "displayName": "NewCluster-1739936324730198", "regionId": "placeholder", "labels": { "tidb.cloud/project": "placeholder" }, "tidbNodeSetting": { "nodeSpecKey": "2C8G", "tidbNodeGroups": [ { "nodeCount": 2 } ] }, "tikvNodeSetting": { "nodeCount": 3, "nodeSpecKey": "2C8G", "storageSizeGi": 500, "storageType": "Standard" }, "port": 4000, "rootPassword": "MySecurePassword123!" } } # Process the input using convert_body processed_body, body_params = _convert_body(test_input["body"]) print("Original input:") print(json.dumps(test_input, indent=2)) print("\nProcessed body:") print(json.dumps(processed_body, indent=2)) print("\nBody parameters:") print(json.dumps(body_params, indent=2))