import copy import re from abc import ABC from enum import Enum from typing import TYPE_CHECKING, Any, Dict, List, Optional from environs import Env if TYPE_CHECKING: from models.affected_element import AffectedElement from models.category import Category from models.enhanced_operation import EnhancedOperation from models.response_predictor import ResponsePredictor from utils.dict_utils import add_key, modify_key, remove_key, set_value from utils.value_generator import ( generate_below_min, generate_exceed_max, generate_invalid_enum, generate_invalid_type, generate_null_value, generate_random_value, ) # import random env = Env() env.read_env(".env") def replace_resource_in_path(path: str, obj_source: str, tgt_source: str) -> str: pattern = r"(?<=/)\b" + re.escape(obj_source) + r"\b(?![^{]*?\})" return re.sub(pattern, tgt_source, path) class StrategyInfo(ABC): def __init__(self, name: str, description: str, value: Optional[str] = None): self.name = name self.description = description self.value = value def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return [ { "path": operation.path, "method": operation.method, "operation_id": operation.operationId, "category": category.name, "affected_element": element.key, "element_type": element.__class__.__name__, "generation_strategy": { "name": self.name, # "description": self.description, "value": "placeholder", }, } ] raise NotImplementedError( "This method should be implemented by subclasses if not dry-run" ) def llm_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: # This method implements the logic from generate_negative_cases_by_llm if dry_run: return [ { "path": operation.path, "method": operation.method, "operation_id": operation.operationId, "category": category.name, "affected_element": element.key, "element_type": element.__class__.__name__, "generation_strategy": { "name": self.name, # "description": self.description, "value": "placeholder", }, } ] raise NotImplementedError("This method should be deprecated if not dry-run") def handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, llm=False, ) -> List[Dict[str, Any]]: # print(f"Strategy: {self.name}") # print(f"Key: {element.key}, {element.__class__.__name__}") # print(f"operation_id: {operation.operationId}") # print(f"category: {category.name}\n") # subclass has its own custom_handler if ( hasattr(self.__class__, "custom_handler") and self.__class__.custom_handler is not StrategyInfo.custom_handler ): return self.custom_handler(operation, category, element, dry_run) if llm: # subclass has its own llm_handler if ( hasattr(self.__class__, "llm_handler") and self.__class__.llm_handler is not StrategyInfo.llm_handler ): return self.llm_handler(operation, category, element, dry_run) # use base class llm_handler return self.llm_handler(operation, category, element, dry_run) # use base class custom_handler return self.custom_handler(operation, category, element, dry_run) @classmethod def from_dict(cls, kv: Dict[str, Any]) -> "StrategyInfo": return cls(kv["name"], kv["description"], kv["value"]) class InvalidTypeStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "request": modified_data, new_value = set_value( new_case["request"], element.key, element.reference, generate_invalid_type, ) if modified_data is None: return [] new_case["request"] = modified_data elif category.value.position == "environment": modified_data, new_value = set_value( new_case["environment"], element.key, element.reference, generate_invalid_type, ) if modified_data is None: return [] new_case["environment"] = modified_data else: raise ValueError(f"Unexpected category position: {category.value.position}") new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": new_value} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Set invalid type for `{element.key}`" return [new_case] class NullValueStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "request": if element.key == "method": new_case[element.key] = None elif element.key.startswith("path."): substring = "/" + element.key[5:] target = new_case["path"] index = target.index(substring) if substring + "/" in target: new_case["path"] = target[:index] + target[index + len(substring) :] else: new_case["path"] = target.replace(substring, "") else: modified_data, _ = set_value( new_case["request"], element.key, element.reference, generate_null_value, ) if modified_data is None: return [] new_case["request"] = modified_data elif category.value.position == "environment": modified_data, _ = set_value( new_case["environment"], element.key, element.reference, generate_null_value, ) if modified_data is None: return [] new_case["environment"] = modified_data else: raise ValueError(f"Unexpected category position: {category.value.position}") new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": element.key} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Set null value for `{element.key}`" return [new_case] class ExceedMaxStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "request": modified_data, new_value = set_value( new_case["request"], element.key, element.reference, generate_exceed_max ) if modified_data is None: return [] new_case["request"] = modified_data elif category.value.position == "environment": modified_data, new_value = set_value( new_case["environment"], element.key, element.reference, generate_exceed_max, ) if modified_data is None: return [] new_case["environment"] = modified_data else: raise ValueError(f"Unexpected category position: {category.value.position}") new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": new_value} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Set value exceeding maximum for `{element.key}`" return [new_case] class BelowMinStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "request": modified_data, new_value = set_value( new_case["request"], element.key, element.reference, generate_below_min ) if modified_data is None: return [] new_case["request"] = modified_data elif category.value.position == "environment": modified_data, new_value = set_value( new_case["environment"], element.key, element.reference, generate_below_min, ) if modified_data is None: return [] new_case["environment"] = modified_data else: raise ValueError(f"Unexpected category position: {category.value.position}") self.value = new_value new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": new_value} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Set value below minimum for `{element.key}`" return [new_case] class InvalidEnumStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "request": modified_data, new_value = set_value( new_case["request"], element.key, element.reference, generate_invalid_enum, ) if modified_data is None: return [] new_case["request"] = modified_data elif category.value.position == "environment": modified_data, new_value = set_value( new_case["environment"], element.key, element.reference, generate_invalid_enum, ) if modified_data is None: return [] new_case["environment"] = modified_data else: raise ValueError(f"Unexpected category position: {category.value.position}") new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": element.key} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Set invalid enum value for `{element.key}`" return [new_case] class RandomValueStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "request": if element.key == "method": new_value = "random" new_case[element.key] = new_value elif element.key.startswith("path."): old_value = element.key[5:] target = new_case["path"] new_value = old_value + str( 666 ) # str(generate_random_value(old_value, None)) # str(random.randint(1, 9)) # disable random to check consistency while refatoring new_case["path"] = replace_resource_in_path( target, old_value, new_value ) else: modified_data, new_value = set_value( new_case["request"], element.key, element.reference, generate_random_value, ) if modified_data is None: return [] new_case["request"] = modified_data elif category.value.position == "environment": modified_data, new_value = set_value( new_case["environment"], element.key, element.reference, generate_random_value, ) if modified_data is None: return [] new_case["environment"] = modified_data else: raise ValueError(f"Unexpected category position: {category.value.position}") new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": new_value} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Set random value for `{element.key}`" return [new_case] class InvalidKeyStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "environment": modified_data, new_key = modify_key( new_case["environment"], element.key, element.reference ) elif category.value.position == "request": modified_data, new_key = modify_key( new_case["request"], element.key, element.reference ) else: raise ValueError(f"Unexpected category position: {category.value.position}") if modified_data is None: # No modification was made, so we don't generate a test case return [] if category.value.position == "environment": new_case["environment"] = modified_data elif category.value.position == "request": new_case["request"] = modified_data new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": new_key} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case[ "description" ] = f"Change the key `{element.key}` randomly to {new_key}" return [new_case] class MissingKeyStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "environment": modified_data, key_removed = remove_key( new_case["environment"], element.key ) elif category.value.position == "request": modified_data, key_removed = remove_key(new_case["request"], element.key) else: raise ValueError(f"Unexpected category position: {category.value.position}") if not key_removed: # No key was removed, so we don't generate a test case return [] if category.value.position == "environment": new_case["environment"] = modified_data elif category.value.position == "request": new_case["request"] = modified_data new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = element.key new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = {"name": self.name, "value": element.key} new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case["description"] = f"Remove the key `{element.key}`" return [new_case] class ExtraKeyStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) if category.value.position == "environment": modified_data, new_key, new_value = add_key( new_case["environment"], element.key, element.reference ) elif category.value.position == "request": modified_data, new_key, new_value = add_key( new_case["request"], element.key, element.reference ) else: raise ValueError(f"Unexpected category position: {category.value.position}") if modified_data is None or new_key is None or new_value is None: return [] if category.value.position == "environment": new_case["environment"] = modified_data elif category.value.position == "request": new_case["request"] = modified_data new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = f"{element.key}" new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = { "name": self.name, "value": f"{new_key}: {new_value}", } new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case[ "description" ] = f"Add an extra key `{new_key}` with value `{new_value}` inside `{element.key}`" return [new_case] # class InvalidMethodStrategyInfo(StrategyInfo): # pass class BusinessEnumStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: cases = [] if category.value.position == "request": if operation.typical_positive_case is not None: if getattr(operation.typical_positive_case, element.key): current_value = getattr( operation.typical_positive_case, element.key ) else: current_value = operation.typical_positive_case.request.get( # type: ignore element.key ) else: current_value = getattr(operation, element.key, None) elif category.value.position == "environment": current_value = ( operation.typical_positive_case.environment.get(element.key, None) # type: ignore if operation.typical_positive_case else None ) else: raise ValueError(f"Unexpected category position: {category.value.position}") valid_values = element.reference.enum for value in valid_values: # type: ignore if str(value).lower() != str(current_value).lower(): self.value = value # current_value c = { "path": operation.path, "method": operation.method, "operation_id": operation.operationId, "category": category.name, "affected_element": element.key, "element_type": element.__class__.__name__, "generation_strategy": { "name": self.name, # "description": self.description, "value": value, }, } if not dry_run: c[ "description" ] = f"Using valid but unexpected enum value '{value}' for {element.key}" c["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) c["environment"] = copy.deepcopy( operation.typical_positive_case.environment # type: ignore ) c["request"] = copy.deepcopy( operation.typical_positive_case.request # type: ignore ) if category.value.position == "request": if c.get(element.key): c[element.key] = value elif c["request"].get(element.key): c["request"][element.key] = value else: raise ValueError(f"Key {element.key} not found in request") elif category.value.position == "environment": c["environment"][element.key] = value cases.append(c) return cases class ForbiddenStrategyInfo(StrategyInfo): def custom_handler( self, operation: "EnhancedOperation", category: "Category", element: "AffectedElement", dry_run: bool = False, ) -> List[Dict[str, Any]]: if dry_run: return super().custom_handler(operation, category, element, dry_run) new_case = copy.deepcopy(operation.typical_positive_case.__dict__) prj_id_403 = env.str("PRJ_ID_403", None) if element.key == "query_params.projectId": new_case["environment"]["prj_id"] = prj_id_403 if new_case["request"].get("query_params"): new_case["request"]["query_params"]["projectId"] = prj_id_403 else: new_case["request"]["query_params"] = {"projectId": prj_id_403} new_case["type"] = "Negative" new_case["category"] = category.name new_case["affected_element"] = f"{element.key}" new_case["element_type"] = element.__class__.__name__ new_case["generation_strategy"] = { "name": self.name, "value": f"projectId: {prj_id_403}", } new_case["expected_status"] = ResponsePredictor.predict( category, element, self, operation ) new_case[ "description" ] = f"Construct `projectId` value to `{prj_id_403}` to assert 403 forbidden" return [new_case] else: raise NotImplementedError( "Forbidden strategy is only implemented for query_params.projectId now" ) class GenerationStrategy(Enum): INVALID_TYPE = InvalidTypeStrategyInfo( "INVALID_TYPE", "Set the given key's value in an invalid type in the positive case", ) NULL_VALUE = NullValueStrategyInfo( "NULL_VALUE", "Set the given key's value to `null` in the positive case" ) EXCEED_MAX = ExceedMaxStrategyInfo( "EXCEED_MAX", "ONLY For int/float/string. For int/float: Set the given key's value to exceed the max value in the positive case; For string: Set the given key's value to `ThisIsAnExceedinglyVeryLongValueThatShouldTriggerAnError` which exceeds the max length in the positive case if the max length is given(DON'T REPEAT the value in positive case but set to `ThisIsAnExceedinglyVeryLongValueThatShouldTriggerAnError`)", ) BELOW_MIN = BelowMinStrategyInfo( "BELOW_MIN", "ONLY For int/float/string. For int/float: Set the given key to below the min value in the positive case; For string: Set the given key to below the min length in the positive case if the min length is given", ) INVALID_ENUM = InvalidEnumStrategyInfo( "INVALID_ENUM", "ONLY For enum. Set an invalid enum value in the positive case" ) RANDOM_VALUE = RandomValueStrategyInfo( "RANDOM_VALUE", "Set the given key's value to a random value in the positive case", ) INVALID_KEY = InvalidKeyStrategyInfo( "INVALID_KEY", "Set the given key into an invalid key in the positive case" ) MISSING_KEY = MissingKeyStrategyInfo( "MISSING_KEY", "delete the WHOLE given key-value in the positive case even if the value is an object", ) EXTRA_KEY = ExtraKeyStrategyInfo( "EXTRA_KEY", "Add an extra key-value to the 1st layer of object in the positive case", ) BUSINESS_ENUM = BusinessEnumStrategyInfo( "BUSINESS_ENUM", "The given enum value is valid but differs the business logic. Just replace it in the positive case.", ) FORBIDDEN_403 = ForbiddenStrategyInfo( "FORBIDDEN_403", "Construct a 403 forbidden response with the given message in the positive case", )