import logging import re from typing import Optional, List from github import Github, Auth from github.Issue import Issue from pydantic import BaseModel from packaging.version import Version from app.config import settings, FetchBugInfoMethod from app.libs.tirelease.tirelease import TiReleaseAPI from app.models.support_ticket import ( MentionGitHubIssueLink, MentionGitHubIssue, BugFixPullRequest, ) from app.writer.tools.github_gql_client import get_github_graphql_client from gql import gql logger = logging.getLogger(__name__) github = Github(auth=Auth.Token(settings.GITHUB_TOKEN)) tirelease = TiReleaseAPI(base_url="https://tirelease.pingcap.net") github_gql_client = get_github_graphql_client() GITHUB_ISSUE_LINK_PATTERN = ( r"https://github\.com/(?P\w+)/(?P\w+)/issues/(?P\d+)" ) def extract_github_issue_links(text: str) -> set[MentionGitHubIssueLink]: links = set() for match in re.finditer(GITHUB_ISSUE_LINK_PATTERN, text): link = MentionGitHubIssueLink( url=match.group(0), owner=match.group("owner"), repo=match.group("repo"), number=int(match.group("number")), ) links.add(link) return links def fetch_mention_github_issues(texts: List[str]) -> list[MentionGitHubIssue]: links = set() for text in texts: if text is None: continue links.update(extract_github_issue_links(text)) mention_github_issues = [] for link in links: try: issue = fetch_github_issue(link) mention_github_issues.append(issue) except Exception as e: logger.error(f"Failed to fetch GitHub issue {link}: {e}") return mention_github_issues def fetch_github_issue(link: MentionGitHubIssueLink) -> MentionGitHubIssue: full_name = f"{link.owner}/{link.repo}" repo = github.get_repo(full_name, lazy=True) github_issue = repo.get_issue(link.number) if repo.owner.login in ["pingcap", "tikv"]: bug_info = fetch_bug_information(github_issue) else: bug_info = {} return MentionGitHubIssue( link=link, title=github_issue.title, state=github_issue.state, body=github_issue.body, labels=[label.name for label in github_issue.labels], **bug_info, ) def fetch_bug_information(issue: Issue) -> dict: if settings.FETCH_BUG_INFO_METHOD == FetchBugInfoMethod.TI_RELEASE: return fetch_bug_information_from_tirelease(issue) else: return fetch_bug_information_from_github(issue) def fetch_bug_information_from_tirelease(issue: Issue) -> dict: try: node_id = issue.raw_data["node_id"] bug_issue = tirelease.get_bug_issue_information(node_id) return { "components": bug_issue["components"], "affect_versions": bug_issue["affect_versions"], "fixed_versions": bug_issue["fixed_versions"], } except Exception as e: logger.error(f"Failed to get bug information from TiRelease: {e}") return {} # Fetch BUG information from GitHub AFFECT_VERSION_LABEL_PATTERN = re.compile(r"^affects-") TIDB_MINOR_VERSIONS = [ "v6.0", "v6.1", "v6.2", "v6.3", "v6.4", "v6.5", "v6.6", "v7.0", "v7.1", "v7.2", "v7.3", "v7.4", "v7.5", "v7.6", "v8.0", "v8.1", "v8.2", "v8.3", "v8.4", "v8.5", "v8.6", "v9.0", "v9.1", "v9.2", "v9.3", "v9.4", "v9.5", "v9.6", ] TIDB_LTS_VERSIONS = ["v6.1", "v6.5", "v7.1", "v7.5", "v8.1", "v8.5"] def fetch_all_release_versions(owner: str, repo: str) -> list[Version]: """ Fetch release versions from GitHub repository. Args: owner: Repository owner repo: Repository name Returns: List of release version tags sorted in descending order """ try: repo = github.get_repo(f"{owner}/{repo}") releases = repo.get_releases() versions = [ Version(release.tag_name) for release in releases if RELEASE_VERSION_PATTERN.match(release.tag_name) ] return sorted(versions, reverse=True) except Exception as e: logger.error(f"Failed to fetch release versions for {owner}/{repo}: {e}") return [] def fetch_cross_referenced_pulls(owner: str, repo: str, number: int) -> list[dict]: """ Fetch referenced pull requests for a given issue using GitHub GraphQL API. Args: owner: GitHub repository owner repo: GitHub repository name number: GitHub issue number Returns: List of cross-referenced pull requests """ query = gql(""" query($owner: String!, $repo: String!, $number: Int!) { repository(owner: $owner, name: $repo) { issue(number: $number) { timelineItems(first: 100, itemTypes: CROSS_REFERENCED_EVENT) { nodes { ... on CrossReferencedEvent { source { ... on PullRequest { title number mergeCommit { oid } baseRefName } } } } } } } rateLimit { cost used remaining } } """) try: variables = {"owner": owner, "repo": repo, "number": number} result = github_gql_client.execute(query, variable_values=variables) rate_limit = result["rateLimit"] logger.info( f"GitHub GraphQL rate limit - Used: {rate_limit['used']}, " f"Remaining: {rate_limit['remaining']}" ) timeline_items = result["repository"]["issue"]["timelineItems"]["nodes"] return [node["source"] for node in timeline_items if node["source"]] except Exception as e: logger.error( f"Failed to fetch reference pulls for issue {owner}/{repo}#{number}: {e}" ) return [] def get_fixed_patch_version( owner: str, repo: str, patch_versions: list[Version], commit_sha: str ) -> Optional[Version]: """ Determine which patch version first included a specific commit. Args: owner: Repository owner repo: Repository name patch_versions: List of version tags to check commit_sha: The commit hash to look for Returns: The first version that includes the commit, or None if not found """ if not patch_versions: return None # Build GraphQL query version_strs = [f"v{str(patch_version)}" for patch_version in patch_versions] version_queries = "\n".join( f"""{underscore_version(version_str)}: ref(qualifiedName: "refs/tags/{version_str}") {{ compare(headRef: "{commit_sha}") {{ aheadBy }} }}""" for version_str in version_strs ) query = gql(f""" query {{ repository(owner: "{owner}", name: "{repo}") {{ {version_queries} }} rateLimit {{ cost used remaining }} }} """) try: result = github_gql_client.execute(query) rate_limit = result["rateLimit"] logger.info( f"GitHub GraphQL rate limit - Used: {rate_limit['used']}, " f"Remaining: {rate_limit['remaining']}" ) # aheadBy == 0 means the commit is included in the version. versions_with_fix = [ version for version, ref in result["repository"].items() if ref["compare"]["aheadBy"] == 0 ] return ( Version(versions_with_fix[0].replace("_", ".").strip()) if versions_with_fix else None ) except Exception as e: logger.error( f"Failed to fetch fixed patch version for commit {commit_sha}: {e}" ) return None RELEASE_BRANCH_NAME_PATTERN = re.compile(r"^release-(\d+.\d+)$") def extract_minor_version_from_branch(branch: str) -> Optional[Version]: match = RELEASE_BRANCH_NAME_PATTERN.match(branch) if match: version = match.group(1) return Version(version) return None def fetch_bug_fix_pulls( owner: str, repo: str, number: int, released_versions: set[Version] ) -> list[BugFixPullRequest]: # Fetch cross-referenced pull requests cross_referenced_pulls = fetch_cross_referenced_pulls(owner, repo, number) logger.info( f"Fetched cross-referenced pull requests for issue {owner}/{repo}#{number}." ) pulls = [] for pull in cross_referenced_pulls: # Notice: only consider the pull request has been merged. if not pull or "mergeCommit" not in pull: continue branch = pull["baseRefName"] merge_commit_sha = pull["mergeCommit"]["oid"] if pull["mergeCommit"] else None minor_version = extract_minor_version_from_branch(branch) if not minor_version: continue patch_versions = [ version for version in released_versions if version.major == minor_version.major and version.minor == minor_version.minor ] print(f"patch_versions: {patch_versions}") fxied_version = get_fixed_patch_version( owner, repo, patch_versions, merge_commit_sha ) has_release = fxied_version in released_versions pr = BugFixPullRequest( title=pull["title"], number=int(pull["number"]), merge_commit_sha=merge_commit_sha, branch=branch, minor_version=str(minor_version), fixed_version=str(fxied_version), has_release=has_release, ) pulls.append(pr) return pulls def fetch_bug_information_from_github(issue: Issue) -> dict: owner = issue.repository.owner.login repo = issue.repository.name number = issue.number labels = [label.name for label in issue.labels] # Extract affected versions affect_versions = extract_affect_versions_from_labels(labels) logger.info( f"Extracted affected versions for issue {owner}/{repo}#{number}: {affect_versions}" ) # Fetch all release versions all_release_versions = fetch_all_release_versions(owner, repo) released_versions = set(all_release_versions) logger.info( f"Fetched all release versions for {owner}/{repo} (total: {len(released_versions)})." ) # Fetch bug fix pull requests fixed_prs = fetch_bug_fix_pulls(owner, repo, number, released_versions) fixed_versions = [pr.fixed_version for pr in fixed_prs] logger.info( f"Fetched bug fix pull requests for issue {owner}/{repo}#{number} (total: {len(fixed_prs)})." ) return { "affect_versions": affect_versions, "fixed_prs": fixed_prs, "fixed_versions": fixed_versions, } def extract_affect_versions_from_labels(labels: List[str]) -> List[str]: """ Extract and sort affected versions from GitHub issue labels. Args: labels: List of GitHub issue labels Returns: Sorted list of version strings """ affected_versions = [] for label in labels: if AFFECT_VERSION_LABEL_PATTERN.match(label): version_str = label.replace("affects-", "") affected_versions.append(version_str) return sorted(affected_versions) def underscore_version(version: str) -> str: return version.replace(".", "_").strip() RELEASE_VERSION_PATTERN = re.compile(r"^v\d+\.\d+\.\d+$")