import json from datetime import datetime, timezone import requests from environs import Env from config.tcms_config import TCMS_CONFIG from src.services import gen_workflow # Load environment variables env = Env() env.read_env() # read .env file, if it exists TCMS_TOKEN = env("TCMS_TOKEN") DEFAULT_CREATOR_EMAIL = "sunyishen@pingcap.com" # harcode now # Determine which host to use based on environment # Set TEST_ENV=local for local testing, otherwise use production TEST_ENV = env("TEST_ENV", "production") # Default to production HOST = TCMS_CONFIG["host_localtest"] if TEST_ENV == "local" else TCMS_CONFIG["host"] BASE_URL = TCMS_CONFIG["base_url"] WORKFLOW_ID = TCMS_CONFIG["workflow_id"] def get_runner_ids(): # GET https://tcms.pingcap.net/api/v1/workflow-runners?offset=0&count=999 return { "aws-us-west-2": 3, "aws-us-east-1": 60002, "gcp-us-west1": 6, "aliyun-ap-southeast-1": 60003, "azure-eastus2": 90001, } def get_workflow(workflow_id): url = f"{HOST}{BASE_URL}/workflows/{workflow_id}" headers = {"Authorization": f"Bearer {TCMS_TOKEN}"} try: response = requests.get(url, headers=headers) response.raise_for_status() print(f"Status Code: {response.status_code}") print("Response JSON:") print(json.dumps(response.json(), indent=2, ensure_ascii=False)) except requests.exceptions.RequestException as e: print(f"Error fetching workflow: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") def create_workflow_execution( workflow_id, name, sku_config, yaml_path="config/test_workflow.yaml", cloud_provider=None, region=None, ): """ Creates a workflow execution by generating the spec from gen_workflow.py and posting it to TCMS API. Args: workflow_id: The workflow ID to execute name: Name for this execution sku_config: SKU config dict (required) yaml_path: Path to workflow YAML file runner_id: Runner ID for execution (default: 3) """ url = f"{HOST}{BASE_URL}/workflow-executions" headers = { "Authorization": f"Bearer {TCMS_TOKEN}", "Content-Type": "application/json", } if not cloud_provider or not region: raise ValueError("cloud_provider and region are required to resolve runner_id") runner_ids = get_runner_ids() # region already stores values like 'aws-us-west-2', so use it directly runner_key = region if runner_key not in runner_ids: raise ValueError(f"Region not found for key: {runner_key}") runner_id = runner_ids[runner_key] # Generate the workflow spec using gen_workflow workflow = gen_workflow.load_workflow(yaml_path) # Remove startNodeParams to regenerate fresh if "startNodeParams" in workflow: del workflow["startNodeParams"] gen_workflow.modify_workflow(workflow, sku_config) workflow = gen_workflow.sync_start_node_params(workflow) # Construct the execution request payload payload = { "workflowID": workflow_id, "name": name, "spec": workflow, "source": "TCMS", "runnerID": runner_id, } # # Mock behavior: write payload to a local JSON file instead of sending HTTP request # file_name = f"workflow_execution_{name}.json" # try: # with open(file_name, "w", encoding="utf-8") as f: # json.dump(payload, f, indent=2, ensure_ascii=False) # print(f"Mock workflow execution request written to {file_name}") # except OSError as e: # print(f"Failed to write mock workflow execution payload: {e}") # # In mock mode we return None so callers treat this as "no resp" and # # therefore will not update task status or workflow_execution_id. # return None # If you want to send the real HTTP request instead of mocking, uncomment below: try: response = requests.post(url, headers=headers, json=payload) response.raise_for_status() print(f"Status Code: {response.status_code}") print("Execution created successfully!") print("Response JSON:") print(json.dumps(response.json(), indent=2, ensure_ascii=False)) return response.json() except requests.exceptions.RequestException as e: print(f"Error creating workflow execution: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") return None def get_execution_status(execution_id): """ Fetches the status of a workflow execution. Args: execution_id: The ID of the workflow execution Returns: dict: The execution status object or None if error """ url = f"{HOST}{BASE_URL}/workflow-executions/{execution_id}" headers = {"Authorization": f"Bearer {TCMS_TOKEN}"} try: response = requests.get(url, headers=headers) response.raise_for_status() data = response.json() # The execution response often includes a full workflow "spec" (large). # Current scheduler/parser logic relies on fields like "status", "output", # and "graph", but not on "spec". Dropping it reduces DB row size pressure # when persisting raw execution responses. if isinstance(data, dict): data.pop("spec", None) return data except requests.exceptions.RequestException as e: print(f"Error fetching execution status for {execution_id}: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") return None def list_workflow_executions( offset: int = 0, count: int = 200, *, creator_email: str = DEFAULT_CREATOR_EMAIL, statuses: tuple[str, ...] = ("PENDING", "RUNNING", "CANCELLING"), ): """Lists workflow executions with pagination.""" url = f"{HOST}{BASE_URL}/workflow-executions" headers = {"Authorization": f"Bearer {TCMS_TOKEN}"} params = { "offset": offset, "count": count, "creator_email": creator_email, "statuses": ",".join(statuses), } try: response = requests.get(url, headers=headers, params=params) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: print(f"Error listing workflow executions: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") return None def find_workflow_execution_by_name( workflow_name: str, *, creator_email: str = DEFAULT_CREATOR_EMAIL, statuses: tuple[str, ...] = ("PENDING", "RUNNING", "CANCELLING"), max_pages: int = 10, page_size: int = 2000, ): """Find a workflow execution by workflowName (the 'name' used in create). Returns: The execution dict if found, otherwise None. """ # If workflow_name ends with a unix timestamp (our naming convention), # use it to stop paging early when the server returns results sorted by createdAt desc. cutoff_dt = None parts = workflow_name.rsplit("_", 1) if len(parts) == 2 and parts[1].isdigit(): try: ts = int(parts[1]) cutoff_dt = datetime.fromtimestamp(ts - 30 * 60, tz=timezone.utc) except Exception: cutoff_dt = None def _parse_created_at(value: str): if not value or not isinstance(value, str): return None try: # Common formats: 2026-01-26T00:00:00Z / with offset return datetime.fromisoformat(value.replace("Z", "+00:00")) except Exception: return None offset = 0 prev_last_dt = None assume_sorted_desc = True for _ in range(max_pages): payload = list_workflow_executions( offset=offset, count=page_size, creator_email=creator_email, statuses=statuses, ) if not payload: return None data = payload.get("data") or [] created_dts = [] for item in data: if item.get("workflowName") == workflow_name: return item created_dt = _parse_created_at(item.get("createdAt")) if created_dt is not None: created_dts.append(created_dt) if assume_sorted_desc and created_dts: # If list is sorted desc, the last item should be oldest in this page. page_last_dt = created_dts[-1] if prev_last_dt is not None and page_last_dt > prev_last_dt: assume_sorted_desc = False prev_last_dt = page_last_dt if cutoff_dt is not None and page_last_dt < cutoff_dt: return None total = payload.get("total") if total is None: if len(data) < page_size: return None offset += page_size continue offset += page_size if offset >= int(total): return None return None def cancel_workflow_execution(execution_id: str) -> bool: """Cancels a workflow execution via TCMS remote-control API. PUT /workflow-executions/{id}/remote-control Body: {"cancel": true} Returns: True if TCMS returns 200, otherwise False. """ url = f"{HOST}{BASE_URL}/workflow-executions/{execution_id}/remote-control" headers = { "Authorization": f"Bearer {TCMS_TOKEN}", "Content-Type": "application/json", } payload = {"cancel": True} try: response = requests.put(url, headers=headers, json=payload) response.raise_for_status() return True except requests.exceptions.RequestException as e: print(f"Error cancelling workflow execution {execution_id}: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") return False def get_case_executions(plan_execution_id, offset=0, count=20): """ Fetches case executions for a plan execution. Args: plan_execution_id: The plan execution ID offset: Offset for pagination (default: 0) count: Number of results to return (default: 20) Returns: dict: Case executions response with 'data' array or None if error """ url = f"{HOST}{BASE_URL}/case-executions" params = { "offset": offset, "count": count, "root_plan_exec_id": plan_execution_id, } headers = {"Authorization": f"Bearer {TCMS_TOKEN}"} try: response = requests.get(url, headers=headers, params=params) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: print(f"Error fetching case executions for plan {plan_execution_id}: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") return None def get_case_execution_log(log_url): """ Fetches the log content from a case execution artifact URL. The log_url should be the original URL from artifactURLs.main-logs. This function will replace the host with the configured TCMS host. Args: log_url: The artifact log URL (e.g., from artifactURLs.main-logs) Returns: str: Log content as text or None if error """ # Replace the host in the URL with our configured HOST # Extract the path after the host if "://" in log_url: # Split on :// to get protocol and rest parts = log_url.split("://", 1) if len(parts) == 2: # Get everything after the first / rest = parts[1] if "/" in rest: path = rest.split("/", 1)[1] # Construct new URL with our HOST new_url = f"{HOST}/{path}" else: new_url = log_url else: new_url = log_url else: new_url = log_url headers = {"Authorization": f"Bearer {TCMS_TOKEN}"} try: response = requests.get(new_url, headers=headers) response.raise_for_status() return response.text except requests.exceptions.RequestException as e: print(f"Error fetching log from {new_url}: {e}") if hasattr(e, "response") and e.response is not None: print(f"Response content: {e.response.text}") return None if __name__ == "__main__": # Example: Get workflow details print(f"Fetching workflow {WORKFLOW_ID}...") get_workflow(WORKFLOW_ID) # # Example: Create a workflow execution with config from config/sku_config.py # from config.sku_config import SKU_CONFIG # print(f"Creating execution for workflow {WORKFLOW_ID}...") # create_workflow_execution( # workflow_id=WORKFLOW_ID, # name="yishen_test_workload", # sku_config=SKU_CONFIG, # yaml_path="config/test_workflow.yaml" # ) # # Example: Create a workflow execution with custom config dict # custom_sku_config = { # "cloud_provider": "aws", # "region": "us-west-2", # "cluster": { # "org_id": "123", # "project_id": "456", # "tidb": {"nodeProfileName": "4 vCPU, 16 GiB", "replicas": 1}, # "tikv": {"nodeProfileName": "4 vCPU, 16 GiB", "replicas": 3} # } # } # create_workflow_execution( # workflow_id=WORKFLOW_ID, # name="yishen_test_workload", # sku_config=custom_sku_config, # yaml_path="config/test_workflow.yaml" # )