#!/usr/bin/env python3 """ E2E API Pilot Runner - Create-First Pattern Run 10 pilot test cases that create their own data instead of relying on seed data. """ import argparse import glob import json import os import re import sys import time from datetime import datetime from typing import Dict, List, Any, Optional # Add parent directory to path for imports sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import config import api class PilotRunner: """Executes E2E API pilot test cases using Create-First pattern.""" def __init__(self, base_path: str = None): self.base_path = base_path or os.path.dirname(os.path.abspath(__file__)) self.cases_dir = os.path.join(self.base_path, "cases-pilot") self.results = [] self.client = api.NexusAPIClient() self.variables = {} # Runtime variables (e.g., created record IDs) self.timestamp = datetime.now().strftime("%Y%m%d%H%M%S") def _substitute_variables(self, obj: Any) -> Any: """ Replace {variable_name}, {timestamp}, etc. with actual values. Also handles runtime variables set via id_var in setup. """ if isinstance(obj, str): # Replace {timestamp} first result = obj.replace("{timestamp}", self.timestamp) # Replace runtime variables def replace_var(match): var_name = match.group(1) return str(self.variables.get(var_name, match.group(0))) result = re.sub(r'\{([^}]+)\}', replace_var, result) return result elif isinstance(obj, dict): return {k: self._substitute_variables(v) for k, v in obj.items()} elif isinstance(obj, list): return [self._substitute_variables(item) for item in obj] return obj def _load_case(self, case_path: str) -> Dict: """Load a single test case JSON file.""" with open(case_path, 'r') as f: return json.load(f) def _run_request(self, method: str, endpoint: str, payload: Any = None) -> tuple: """Execute HTTP request and return (status_code, response_json, result_id).""" url = f"{self.client.base_url}{endpoint}" headers = {"Authorization": f"Bearer {self.client.token}", "Content-Type": "application/json"} if method == "GET": resp = self.client.session.get(url, headers=headers) elif method == "POST": resp = self.client.session.post(url, headers=headers, json=payload) elif method == "PATCH": resp = self.client.session.patch(url, headers=headers, json=payload) elif method == "DELETE": resp = self.client.session.delete(url, headers=headers) else: raise ValueError(f"Unsupported method: {method}") try: resp_json = resp.json() except: resp_json = {} # Extract ID from response (Salesforce-style success response) result_id = None if resp.status_code in (200, 201) and isinstance(resp_json, dict): result_id = resp_json.get("id") or resp_json.get("Id") return resp.status_code, resp_json, result_id def _execute_setup(self, setup_steps: List[Dict]) -> bool: """Execute setup steps (create prerequisite records).""" if not setup_steps: return True print(f" [setup] Executing {len(setup_steps)} setup step(s)...") for step in setup_steps: object_name = step.get("object", "Unknown") method = step.get("method", "POST") endpoint = self._substitute_variables(step.get("endpoint", "")) payload = self._substitute_variables(step.get("payload", {})) id_var = step.get("id_var") print(f" [setup] Creating {object_name}...") status_code, resp_json, result_id = self._run_request(method, endpoint, payload) if status_code not in (200, 201): print(f" [setup] FAILED: {object_name} creation returned {status_code}") return False if id_var and result_id: self.variables[id_var] = result_id print(f" [setup] Stored {id_var} = {result_id}") return True def _execute_teardown(self, teardown_steps: Any) -> None: """Execute teardown steps (delete created records).""" if not teardown_steps: return # Normalize to list if isinstance(teardown_steps, dict): teardown_steps = [teardown_steps] print(f" [teardown] Executing {len(teardown_steps)} teardown step(s)...") for step in teardown_steps: method = step.get("method", "DELETE") endpoint = self._substitute_variables(step.get("endpoint", "")) print(f" [teardown] {method} {endpoint}") try: self._run_request(method, endpoint) except Exception as e: print(f" [teardown] Warning: {e}") def _run_case(self, case: Dict) -> Dict: """Execute a single test case.""" case_id = case.get("id", "UNKNOWN") case_name = case.get("name", "Unnamed Test") scene = case.get("scene", "unknown") print(f"\n{'='*60}") print(f"Running: {case_id} - {case_name}") print(f"Scene: {scene}") print(f"{'='*60}") result = { "id": case_id, "name": case_name, "scene": scene, "status": "RUNNING", "started_at": datetime.now().isoformat(), } try: # Reset variables for this case self.variables = {} # Execute setup first (Create-First pattern) setup_steps = case.get("setup", []) if not self._execute_setup(setup_steps): result["status"] = "SETUP_FAILED" result["ended_at"] = datetime.now().isoformat() return result # Execute main request method = case.get("method", "GET") endpoint = self._substitute_variables(case.get("endpoint", "")) payload = self._substitute_variables(case.get("payload", {})) print(f" [test] {method} {endpoint}") if payload: print(f" [test] Payload: {json.dumps(payload, indent=4)}") status_code, resp_json, result_id = self._run_request(method, endpoint, payload) print(f" [test] Response: {status_code}") # Store result_id if available if result_id: self.variables["result_id"] = result_id print(f" [test] Created record ID: {result_id}") # Extract fields from response for verification if isinstance(resp_json, dict): for field in ["id", "Id", "Name", "LastName", "Status"]: if field in resp_json and field.lower() != "id" or (isinstance(resp_json.get(field), str)): var_key = f"result_{field}" if field != "id" else "result_id" self.variables[var_key] = str(resp_json.get(field, resp_json.get(field.lower(), ""))) # Check expectations expected_status = case.get("expects", {}).get("status_code", 200) if status_code != expected_status: result["status"] = "FAILED" result["reason"] = f"Status code {status_code} != expected {expected_status}" result["response"] = resp_json print(f" [test] FAILED: {result['reason']}") self._execute_teardown(case.get("teardown")) result["ended_at"] = datetime.now().isoformat() return result # Execute query_after if present query_after = case.get("query_after") if query_after: query_object = query_after.get("object", "") query_id = self._substitute_variables(query_after.get("id", "{result_id}")) assertions = query_after.get("assertions", []) print(f" [verify] Querying {query_object} with id={query_id}") print(f" [verify] Running {len(assertions)} assertion(s)") query_url = f"{self.client.base_url}/services/data/v63.0/sobjects/{query_object}/{query_id}" headers = {"Authorization": f"Bearer {self.client.token}"} resp = self.client.session.get(query_url, headers=headers) if resp.status_code != 200: result["status"] = "FAILED" result["reason"] = f"Query returned {resp.status_code}" print(f" [verify] FAILED: {result['reason']}") self._execute_teardown(case.get("teardown")) result["ended_at"] = datetime.now().isoformat() return result record = resp.json() # Run assertions for assertion in assertions: field = assertion.get("field", "") operator = assertion.get("operator", "eq") expected_value = self._substitute_variables(assertion.get("value")) actual_value = record.get(field) # Apply operator check passed = False if operator == "eq": passed = str(actual_value) == str(expected_value) elif operator == "ne": passed = str(actual_value) != str(expected_value) elif operator == "contains": passed = str(expected_value) in str(actual_value) elif operator == "not_contains": passed = str(expected_value) not in str(actual_value) elif operator == "is_null": passed = actual_value is None or actual_value == "" elif operator == "is_not_null": passed = actual_value is not None and actual_value != "" elif operator == "gt": passed = float(actual_value or 0) > float(expected_value) status_str = "PASS" if passed else "FAIL" print(f" [verify] {status_str}: {field} {operator} {expected_value} (actual: {actual_value})") if not passed: result["status"] = "FAILED" result["reason"] = f"Assertion failed: {field} {operator} {expected_value} (actual: {actual_value})" print(f" [verify] FAILED: {result['reason']}") self._execute_teardown(case.get("teardown")) result["ended_at"] = datetime.now().isoformat() return result # All checks passed result["status"] = "PASSED" print(f" [test] PASSED!") # Execute teardown self._execute_teardown(case.get("teardown")) except Exception as e: result["status"] = "ERROR" result["reason"] = str(e) print(f" [test] ERROR: {e}") result["ended_at"] = datetime.now().isoformat() return result def run(self, case_pattern: str = None) -> List[Dict]: """Run all pilot cases or a specific case pattern.""" # Find case files if case_pattern: case_files = glob.glob(os.path.join(self.cases_dir, f"*{case_pattern}*.json")) else: case_files = glob.glob(os.path.join(self.cases_dir, "*.json")) if not case_files: print(f"No cases found matching pattern: {case_pattern or '*'}") return [] print(f"\nFound {len(case_files)} case(s) to run") print(f"Timestamp for this run: {self.timestamp}") # Authenticate print("\nAuthenticating...") if not self.client.authenticate(): print("Authentication failed!") return [] # Run each case for case_file in sorted(case_files): case = self._load_case(case_file) result = self._run_case(case) self.results.append(result) # Print summary self._print_summary() return self.results def _print_summary(self) -> None: """Print test run summary.""" print(f"\n{'='*60}") print("SUMMARY") print(f"{'='*60}") total = len(self.results) passed = sum(1 for r in self.results if r["status"] == "PASSED") failed = sum(1 for r in self.results if r["status"] == "FAILED") errors = sum(1 for r in self.results if r["status"] in ("ERROR", "SETUP_FAILED")) print(f"Total: {total}") print(f"Passed: {passed}") print(f"Failed: {failed}") print(f"Errors: {errors}") print(f"Pass Rate: {100*passed/total:.1f}%" if total > 0 else "N/A") print(f"\n{'='*60}") print("DETAILED RESULTS") print(f"{'='*60}") for r in self.results: status_symbol = "✓" if r["status"] == "PASSED" else "✗" print(f" {status_symbol} {r['id']}: {r['status']} - {r.get('reason', '')}") # Save report report_file = os.path.join(self.base_path, f"pilot-report-{self.timestamp}.json") with open(report_file, 'w') as f: json.dump({ "timestamp": self.timestamp, "summary": { "total": total, "passed": passed, "failed": failed, "errors": errors, "pass_rate": f"{100*passed/total:.1f}%" if total > 0 else "N/A" }, "results": self.results }, f, indent=2) print(f"\nReport saved to: {report_file}") def main(): parser = argparse.ArgumentParser(description="Run E2E API Pilot Cases (Create-First Pattern)") parser.add_argument("--case", "-c", help="Run specific case ID (e.g., LEAD-001)") args = parser.parse_args() runner = PilotRunner() results = runner.run(case_pattern=args.case) # Exit with error code if any tests failed if any(r["status"] != "PASSED" for r in results): sys.exit(1) if __name__ == "__main__": main()