#!/usr/bin/env python3 """Migrate TiDB Cloud Jira Assets fields to Forge object custom fields. The migration is intentionally dry-run by default. Pass ``--apply`` only after reviewing the emitted per-issue plan and summary. """ from __future__ import annotations import argparse import base64 import json import os import re import sys import time import urllib.error import urllib.parse import urllib.request from collections import Counter from dataclasses import dataclass from typing import Any, Callable, Iterable DEFAULT_PROJECTS = ("TCID","ST") DEFAULT_LEGACY_ORG_FIELD = "customfield_10086" DEFAULT_LEGACY_CLUSTER_FIELD = "customfield_10088" ASSET_BUSINESS_ID_ATTRIBUTE = "ID" JSM_API_PATH = "/api/v1/jsm-syncer" CUSTOM_FIELD_PATTERN = re.compile(r"customfield_(\d+)") PROJECT_PATTERN = re.compile(r"[A-Z][A-Z0-9_]*") RETRYABLE_HTTP_STATUSES = {429, 500, 502, 503, 504} class MigrationError(RuntimeError): """An expected migration failure with an operator-actionable message.""" class APIError(MigrationError): """An HTTP API request failed.""" def basic_auth_header(username: str, token: str) -> str: raw = f"{username}:{token}".encode("utf-8") return "Basic " + base64.b64encode(raw).decode("ascii") class JSONHTTPClient: def __init__(self, timeout: float, retries: int) -> None: self.timeout = timeout self.retries = retries def request( self, method: str, url: str, *, headers: dict[str, str] | None = None, payload: Any = None, expected_statuses: tuple[int, ...] = (200,), ) -> Any: body = None display_url = url.split("?", 1)[0] request_headers = {"Accept": "application/json", **(headers or {})} if payload is not None: body = json.dumps(payload, separators=(",", ":")).encode("utf-8") request_headers["Content-Type"] = "application/json" for attempt in range(self.retries + 1): request = urllib.request.Request( url, data=body, method=method, headers=request_headers, ) try: with urllib.request.urlopen(request, timeout=self.timeout) as response: raw = response.read() if response.status not in expected_statuses: raise APIError( f"{method} {display_url} returned unexpected HTTP {response.status}" ) if not raw: return None try: return json.loads(raw.decode("utf-8")) except json.JSONDecodeError as exc: raise APIError( f"{method} {display_url} returned invalid JSON" ) from exc except urllib.error.HTTPError as exc: response_body = exc.read().decode("utf-8", errors="replace") if exc.code in RETRYABLE_HTTP_STATUSES and attempt < self.retries: retry_after = exc.headers.get("Retry-After") if exc.headers else None self._sleep_before_retry(attempt, retry_after) continue raise APIError( f"{method} {display_url} failed with HTTP {exc.code}: " f"{response_body[:4096]}" ) from exc except urllib.error.URLError as exc: if attempt < self.retries: self._sleep_before_retry(attempt, None) continue raise APIError(f"{method} {display_url} failed: {exc.reason}") from exc raise AssertionError("HTTP retry loop exited unexpectedly") @staticmethod def _sleep_before_retry(attempt: int, retry_after: str | None) -> None: try: delay = float(retry_after) if retry_after else min(2**attempt, 30) except ValueError: delay = min(2**attempt, 30) time.sleep(max(delay, 0)) class JiraClient: def __init__( self, base_url: str, username: str, token: str, cloud_id: str, http: JSONHTTPClient, ) -> None: self.base_url = base_url.rstrip("/") self.auth_headers = {"Authorization": basic_auth_header(username, token)} self.cloud_id = cloud_id.strip() self.http = http self._asset_objects: dict[tuple[str, str], dict[str, Any]] = {} self._asset_attribute_names: dict[tuple[str, str], dict[str, str]] = {} def search_issues( self, jql: str, fields: list[str], page_size: int, ) -> Iterable[dict[str, Any]]: next_page_token = "" while True: payload: dict[str, Any] = { "jql": jql, "fields": fields, "maxResults": page_size, } if next_page_token: payload["nextPageToken"] = next_page_token page = self.http.request( "POST", self.base_url + "/rest/api/3/search/jql", headers=self.auth_headers, payload=payload, ) issues = page.get("issues") if isinstance(page, dict) else None if not isinstance(issues, list): raise APIError("Jira enhanced search response is missing an issues array") yield from issues next_page_token = page.get("nextPageToken") or "" if page.get("isLast") is True or not issues or not next_page_token: return def update_issue(self, issue_key: str, fields: dict[str, Any]) -> None: key = urllib.parse.quote(issue_key, safe="") self.http.request( "PUT", f"{self.base_url}/rest/api/3/issue/{key}", headers=self.auth_headers, payload={"fields": fields}, expected_statuses=(204,), ) def get_asset_object(self, workspace_id: str, object_id: str) -> dict[str, Any]: cache_key = (workspace_id, object_id) if cache_key in self._asset_objects: return self._asset_objects[cache_key] cloud_id = self.cloud_id or self._discover_cloud_id() base = ( "https://api.atlassian.com/ex/jira/" f"{urllib.parse.quote(cloud_id, safe='')}/jsm/assets/workspace/" f"{urllib.parse.quote(workspace_id, safe='')}/v1" ) asset = self.http.request( "GET", f"{base}/object/{urllib.parse.quote(object_id, safe='')}", headers=self.auth_headers, ) if not isinstance(asset, dict): raise APIError(f"Assets object {workspace_id}:{object_id} is not an object") self._enrich_asset_attribute_names(base, asset) self._asset_objects[cache_key] = asset return asset def _discover_cloud_id(self) -> str: response = self.http.request( "GET", self.base_url + "/_edge/tenant_info", headers=self.auth_headers, ) cloud_id = response.get("cloudId") if isinstance(response, dict) else "" if not cloud_id: raise APIError("could not discover Jira cloudId; set --jira-cloud-id") self.cloud_id = str(cloud_id) return self.cloud_id def _enrich_asset_attribute_names(self, asset_base: str, asset: dict[str, Any]) -> None: attributes = asset.get("attributes") if not isinstance(attributes, list) or all(attribute_name(item) for item in attributes): return object_type = asset.get("objectType") or {} object_type_id = str(object_type.get("id") or asset.get("objectTypeId") or "") if not object_type_id: return cache_key = (asset_base, object_type_id) names = self._asset_attribute_names.get(cache_key) if names is None: definitions = self.http.request( "GET", f"{asset_base}/objecttype/{urllib.parse.quote(object_type_id, safe='')}/attributes", headers=self.auth_headers, ) if not isinstance(definitions, list): raise APIError(f"Assets object type {object_type_id} attributes are invalid") names = { str(item.get("id")): str(item.get("name")) for item in definitions if isinstance(item, dict) and item.get("id") and item.get("name") } self._asset_attribute_names[cache_key] = names for attribute in attributes: if not isinstance(attribute, dict) or attribute_name(attribute): continue attribute_id = str(attribute.get("objectTypeAttributeId") or "") if names.get(attribute_id): attribute["_migration_attribute_name"] = names[attribute_id] class JSMSyncerClient: def __init__( self, base_url: str, auth_token: str, http: JSONHTTPClient, ) -> None: normalized = base_url.rstrip("/") self.base_url = ( normalized if normalized.endswith(JSM_API_PATH) else normalized + JSM_API_PATH ) self.http = http self.default_headers = ( {"Authorization": f"Bearer {auth_token}"} if auth_token else {} ) self._orgs: dict[str, dict[str, Any]] = {} self._clusters: dict[str, dict[str, Any]] = {} def find_org(self, org_id: str) -> dict[str, Any]: if org_id in self._orgs: return self._orgs[org_id] path = f"/orgs/{urllib.parse.quote(org_id, safe='')}" response = self._get(path) org = response.get("org") if not isinstance(org, dict): raise APIError("jsm-syncer org response is missing an org object") if str(org.get("tidbcloud_org_id") or "") != org_id: raise MigrationError( f"jsm-syncer returned org {org.get('tidbcloud_org_id')} for requested org {org_id}" ) self._orgs[org_id] = org return org def find_cluster(self, cluster_id: str) -> dict[str, Any]: if cluster_id in self._clusters: return self._clusters[cluster_id] path = f"/clusters/{urllib.parse.quote(cluster_id, safe='')}" response = self._get(path) cluster = response.get("cluster") if not isinstance(cluster, dict): raise APIError("jsm-syncer cluster response is missing a cluster object") if str(cluster.get("id") or "") != cluster_id: raise MigrationError( f"jsm-syncer returned cluster {cluster.get('id')} " f"for requested cluster {cluster_id}" ) self._clusters[cluster_id] = cluster return cluster def _get(self, path: str) -> dict[str, Any]: response = self.http.request( "GET", self.base_url + path, headers=self.default_headers, ) if not isinstance(response, dict): raise APIError(f"jsm-syncer {path} returned a non-object response") if response.get("errcode") not in (None, 0, 200): raise APIError( f"jsm-syncer {path} failed: {response.get('errcode')} {response.get('errmsg', '')}" ) return response def attribute_name(attribute: Any) -> str: if not isinstance(attribute, dict): return "" object_type_attribute = attribute.get("objectTypeAttribute") or {} return str( attribute.get("_migration_attribute_name") or object_type_attribute.get("name") or attribute.get("objectTypeAttributeName") or "" ) def asset_reference(value: Any) -> tuple[str, str] | None: if not isinstance(value, dict): return None workspace_id = str(value.get("workspaceId") or value.get("workspace_id") or "") object_id = str(value.get("objectId") or value.get("object_id") or "") global_id = str(value.get("globalId") or value.get("id") or "") if not object_id and workspace_id and global_id.startswith(workspace_id + ":"): object_id = global_id[len(workspace_id) + 1 :] if workspace_id and object_id: return workspace_id, object_id return None def values_for_attribute(attribute: Any) -> Any: if not isinstance(attribute, dict): return None return attribute.get("objectAttributeValues") or attribute.get("values") or [] def normalize_business_id(value: Any) -> str: if value is None or isinstance(value, bool): return "" if isinstance(value, int): return str(value) if isinstance(value, float) and value.is_integer(): return str(int(value)) if isinstance(value, str): return value.strip() return "" def business_ids_from_attribute_values(value: Any) -> set[str]: values = value if isinstance(value, list) else [value] result: set[str] = set() for item in values: candidate = item if isinstance(item, dict): candidate = item.get("value") if candidate in (None, ""): candidate = item.get("searchValue") if candidate in (None, ""): candidate = item.get("displayValue") normalized = normalize_business_id(candidate) if normalized: result.add(normalized) return result class AssetBusinessIDResolver: def __init__( self, object_loader: Callable[[str, str], dict[str, Any]], ) -> None: self.object_loader = object_loader def resolve(self, raw_value: Any, kind: str) -> str: if kind not in ("org", "cluster"): raise ValueError(f"unsupported Assets value kind: {kind}") items = raw_value if isinstance(raw_value, list) else [raw_value] items = [item for item in items if item not in (None, "")] if len(items) != 1: raise MigrationError( f"legacy {kind} Assets field must contain exactly one value, found {len(items)}" ) item = items[0] reference = asset_reference(item) if reference: asset = self.object_loader(*reference) attributes = asset.get("attributes") or [] if not isinstance(attributes, list): raise MigrationError( f"Assets object for {kind} has an invalid attributes value" ) id_attributes = [ attribute for attribute in attributes if attribute_name(attribute).strip() == ASSET_BUSINESS_ID_ATTRIBUTE ] available_names = sorted( { attribute_name(attribute).strip() or f"" for attribute in attributes if isinstance(attribute, dict) } ) if len(id_attributes) != 1: available = ", ".join(available_names) or "" raise MigrationError( f"Assets object for {kind} must contain exactly one " f"{ASSET_BUSINESS_ID_ATTRIBUTE} attribute, found " f"{len(id_attributes)}; available attributes: {available}" ) raw_id_value = values_for_attribute(id_attributes[0]) resolved_ids = business_ids_from_attribute_values(raw_id_value) if len(resolved_ids) == 1: return next(iter(resolved_ids)) raise MigrationError( f"Assets object {kind} attribute {ASSET_BUSINESS_ID_ATTRIBUTE} " f"must contain exactly one TiDB Cloud ID, found " f"{len(resolved_ids)}; raw value: {str(raw_id_value)[:500]}" ) direct_id = normalize_business_id(item) if direct_id: return direct_id raise MigrationError( f"legacy {kind} field has neither a business ID nor an Assets reference" ) @dataclass(frozen=True) class FieldConfig: legacy_org: str legacy_cluster: str forge_org: str forge_cluster: str def search_fields(self) -> list[str]: fields = [self.legacy_org, self.legacy_cluster] fields.extend(field for field in (self.forge_org, self.forge_cluster) if field) return list(dict.fromkeys(fields)) class MigrationRunner: def __init__( self, jira: JiraClient, jsm: JSMSyncerClient, asset_ids: AssetBusinessIDResolver, fields: FieldConfig, *, apply: bool, overwrite: bool, ) -> None: self.jira = jira self.jsm = jsm self.asset_ids = asset_ids self.fields = fields self.apply = apply self.overwrite = overwrite def process(self, issue: dict[str, Any]) -> dict[str, Any]: issue_key = str(issue.get("key") or "") fields = issue.get("fields") or {} if not issue_key or not isinstance(fields, dict): raise MigrationError("Jira issue response is missing key or fields") updates: dict[str, Any] = {} source_ids: dict[str, str] = {} errors: dict[str, str] = {} org_source = fields.get(self.fields.legacy_org) cluster_source = fields.get(self.fields.legacy_cluster) migrate_org = self._should_migrate(fields, self.fields.forge_org, org_source) migrate_cluster = self._should_migrate( fields, self.fields.forge_cluster, cluster_source ) if not migrate_org and not migrate_cluster: return {"issue": issue_key, "status": "skipped", "source": source_ids} if migrate_org: try: org_id = self.asset_ids.resolve(org_source, "org") source_ids["org_id"] = org_id updates[self.fields.forge_org] = self.jsm.find_org(org_id) except MigrationError as exc: errors["org"] = str(exc) if migrate_cluster: try: cluster_id = self.asset_ids.resolve(cluster_source, "cluster") source_ids["cluster_id"] = cluster_id updates[self.fields.forge_cluster] = self.jsm.find_cluster(cluster_id) except MigrationError as exc: errors["cluster"] = str(exc) if not updates: status = "error" if errors else "skipped" elif self.apply: try: self.jira.update_issue(issue_key, updates) status = "updated_with_errors" if errors else "updated" except MigrationError as exc: errors["jira_update"] = str(exc) status = "error" else: status = "would_update_with_errors" if errors else "would_update" result = { "issue": issue_key, "status": status, "source": source_ids, "target_fields": sorted(updates), } if errors: result["errors"] = errors return result def _should_migrate( self, issue_fields: dict[str, Any], target_field: str, source_value: Any, ) -> bool: if not target_field or source_value in (None, "", []): return False return self.overwrite or issue_fields.get(target_field) in (None, "", []) def custom_field_jql(field: str) -> str: match = CUSTOM_FIELD_PATTERN.fullmatch(field) if not match: raise ValueError(f"invalid Jira custom field ID: {field}") return f"cf[{match.group(1)}]" def build_jql( projects: tuple[str, ...], fields: FieldConfig, issue_key: str, *, overwrite: bool = False, ) -> str: if not projects or any(not PROJECT_PATTERN.fullmatch(project) for project in projects): raise ValueError("projects must be comma-separated Jira project keys") source_clauses = [] if fields.forge_org: source_clauses.append( migration_jql_clause(fields.legacy_org, fields.forge_org, overwrite) ) if fields.forge_cluster: source_clauses.append( migration_jql_clause( fields.legacy_cluster, fields.forge_cluster, overwrite, ) ) if not source_clauses: raise ValueError("at least one Forge target custom field must be configured") project_clause = ", ".join(projects) jql = f"project IN ({project_clause}) AND ({' OR '.join(source_clauses)})" if issue_key: if not re.fullmatch(r"[A-Z][A-Z0-9_]*-\d+", issue_key): raise ValueError("--issue-key must be a Jira issue key") jql += f" AND key = {issue_key}" return jql + " ORDER BY key ASC" def migration_jql_clause(legacy_field: str, forge_field: str, overwrite: bool) -> str: legacy_not_empty = f"{custom_field_jql(legacy_field)} IS NOT EMPTY" if overwrite: return legacy_not_empty return ( f"({legacy_not_empty} AND " f"{custom_field_jql(forge_field)} IS EMPTY)" ) def custom_field(value: str, option_name: str, *, required: bool) -> str: value = value.strip() if not value and not required: return "" if not CUSTOM_FIELD_PATTERN.fullmatch(value): raise ValueError(f"{option_name} must look like customfield_12345") return value def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--jira-url", default=os.getenv("JIRA_URL", "")) parser.add_argument("--jira-username", default=os.getenv("JIRA_USERNAME", "")) parser.add_argument( "--jira-token", default=os.getenv("JIRA_API_TOKEN") or os.getenv("JIRA_PASSWORD", ""), ) parser.add_argument("--jira-cloud-id", default=os.getenv("JIRA_CLOUD_ID", "")) parser.add_argument("--jsm-syncer-url", default=os.getenv("JSM_SYNCER_URL", "")) parser.add_argument( "--jsm-auth-token", default=os.getenv("JSM_SYNCER_AUTH_TOKEN", ""), ) parser.add_argument( "--projects", default=os.getenv("JIRA_PROJECTS", ",".join(DEFAULT_PROJECTS)), help="comma-separated project keys (default: TCID,ST)", ) parser.add_argument( "--legacy-org-field", default=os.getenv("JIRA_LEGACY_ORG_FIELD", DEFAULT_LEGACY_ORG_FIELD), ) parser.add_argument( "--legacy-cluster-field", default=os.getenv("JIRA_LEGACY_CLUSTER_FIELD", DEFAULT_LEGACY_CLUSTER_FIELD), ) parser.add_argument( "--forge-org-field", default=os.getenv("JIRA_FORGE_ORG_FIELD", ""), help="new Forge org field ID, for example customfield_12345", ) parser.add_argument( "--forge-cluster-field", default=os.getenv("JIRA_FORGE_CLUSTER_FIELD", ""), help="new Forge cluster field ID, for example customfield_12346", ) parser.add_argument("--issue-key", default="", help="canary: migrate only one issue") parser.add_argument("--page-size", type=int, default=100) parser.add_argument( "--limit", type=int, default=0, help="stop after N scanned issues; 0 means all", ) parser.add_argument("--timeout", type=float, default=30) parser.add_argument("--retries", type=int, default=4) parser.add_argument("--overwrite", action="store_true", help="replace non-empty Forge values") parser.add_argument("--apply", action="store_true", help="write Jira; default is dry-run") args = parser.parse_args(argv) missing = [ name for name, value in ( ("--jira-url/JIRA_URL", args.jira_url), ("--jira-username/JIRA_USERNAME", args.jira_username), ("--jira-token/JIRA_API_TOKEN", args.jira_token), ("--jsm-syncer-url/JSM_SYNCER_URL", args.jsm_syncer_url), ) if not value ] if missing: parser.error("missing configuration: " + ", ".join(missing)) if not args.forge_org_field and not args.forge_cluster_field: parser.error("configure --forge-org-field and/or --forge-cluster-field") if not 1 <= args.page_size <= 100: parser.error("--page-size must be between 1 and 100") if args.limit < 0 or args.retries < 0 or args.timeout <= 0: parser.error("--limit/--retries must be non-negative and --timeout must be positive") return args def print_json(value: Any, *, stream: Any = sys.stdout) -> None: print(json.dumps(value, ensure_ascii=False, sort_keys=True), file=stream, flush=True) def run(args: argparse.Namespace) -> int: fields = FieldConfig( legacy_org=custom_field(args.legacy_org_field, "--legacy-org-field", required=True), legacy_cluster=custom_field( args.legacy_cluster_field, "--legacy-cluster-field", required=True ), forge_org=custom_field(args.forge_org_field, "--forge-org-field", required=False), forge_cluster=custom_field( args.forge_cluster_field, "--forge-cluster-field", required=False ), ) projects = tuple( item.strip().upper() for item in args.projects.split(",") if item.strip() ) jql = build_jql( projects, fields, args.issue_key.strip().upper(), overwrite=args.overwrite, ) http = JSONHTTPClient(args.timeout, args.retries) jira = JiraClient( args.jira_url, args.jira_username, args.jira_token, args.jira_cloud_id, http, ) jsm = JSMSyncerClient(args.jsm_syncer_url, args.jsm_auth_token, http) asset_ids = AssetBusinessIDResolver(jira.get_asset_object) runner = MigrationRunner( jira, jsm, asset_ids, fields, apply=args.apply, overwrite=args.overwrite, ) print_json( { "mode": "apply" if args.apply else "dry-run", "jql": jql, "forge_org_field": fields.forge_org or None, "forge_cluster_field": fields.forge_cluster or None, } ) counts: Counter[str] = Counter() error_counts: Counter[str] = Counter() had_errors = False scanned = 0 for issue in jira.search_issues( jql, fields.search_fields(), args.page_size, ): if args.limit and scanned >= args.limit: break scanned += 1 issue_key = str(issue.get("key") or "") try: result = runner.process(issue) counts[result["status"]] += 1 result_errors = result.get("errors") if isinstance(result_errors, dict) and result_errors: had_errors = True error_counts.update(str(kind) for kind in result_errors) print_json(result, stream=sys.stderr) else: print_json(result) except MigrationError as exc: had_errors = True counts["error"] += 1 error_counts["issue"] += 1 print_json( { "issue": issue_key, "status": "error", "errors": {"issue": str(exc)}, }, stream=sys.stderr, ) summary = { "mode": "apply" if args.apply else "dry-run", "scanned": scanned, "counts": dict(sorted(counts.items())), "error_counts": dict(sorted(error_counts.items())), } print_json({"summary": summary}) return 1 if had_errors else 0 def main(argv: list[str] | None = None) -> int: try: return run(parse_args(argv)) except (APIError, ValueError) as exc: print_json({"status": "fatal", "error": str(exc)}, stream=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())