import configparser import json import os import sys import yaml ROOT = sys.path[0] + "/../" class PulumiConfig: def __init__(self, pulumi_path: str, service_names: list[str], repo): self.pulumi_path = pulumi_path self.service_names = service_names self.repo = repo def to_list(self) -> list[dict[str, str]]: result = [] for service_name in self.service_names: result.append( { "service_name": service_name, "repo": self.repo.url, "pulumi_path": self.pulumi_path, "branch": self.repo.branch, } ) return result class SubRepo: def __init__(self, path: str = "", url: str = "", branch: str = ""): self.path = path self.url = url self.branch = branch def parse_pulumi_config(self) -> list[PulumiConfig]: result = [] for root, _, files in os.walk(ROOT + self.path): if "Pulumi.yaml" not in files: continue _, parent = root.split(self.path) for f in files: if not f.endswith(".ts"): continue output = os.popen( f"node {ROOT}/parser/pulumi/pulumi2service.ts {root}/{f}" ) # Specail repos that use variable as Deployment name if self.path.endswith("internal-portal"): service_names = ["internal-portal-server"] else: service_names = ( output.read().replace('"', "").replace("'", "").split("\n") ) service_names = list(filter(lambda v: v != "", service_names)) if len(service_names) == 0: continue result.append(PulumiConfig(f"{parent}/{f}", service_names, self)) return result def parse_proto(self) -> list[dict]: result = [] for root, _, files in os.walk(ROOT + self.path): files = list(filter(lambda f: f.endswith(".proto"), files)) if not files: continue _, parent = root.split(self.path) for f in files: output = os.popen(f"go run {ROOT}/parser/proto/main.go -p {root}/{f}") rpcs = yaml.safe_load(output.read()) if rpcs is None: continue result.append( { "proto_path": f"{parent}/{f}", "repo": self.url, "rpcs": rpcs, } ) return result def parse_swagger_http(self): result = [] for root, _, files in os.walk(ROOT + self.path): files = list( filter( lambda f: "swagger" in f and (f.endswith("json") or f.endswith("yaml")), files, ) ) if not files: continue if len(files) == 2: concat_files = files[0] + files[1] if "yaml" in concat_files and "json" in concat_files: files = list(filter(lambda f: f.endswith("json"), files)) _, parent = root.split(self.path) for f in files: content = {} if f.endswith("json"): with open(f"{root}/{f}") as sf: content = json.loads(sf.read()) elif f.endswith("yaml"): with open(f"{root}/{f}") as sf: content = yaml.safe_load(sf.read()) if content["swagger"] != "2.0": raise ValueError("Invalid swagger version") base_path = content.get("basePath", "") apis = [] paths = content.get("paths", {}) for path, methods in paths.items(): for method in methods.keys(): apis.append( {"path": f"{base_path}{path}", "method": method.upper()} ) result.append( { "swagger_path": f"{parent}/{f}", "repo": self.url, "apis": apis, } ) return result def list_sub_repos() -> list[SubRepo]: p = configparser.ConfigParser() p.read(f"{ROOT}/.gitmodules", encoding="utf-8") result = [] for section in p.sections(): repo = SubRepo() for item in p.items(section): setattr(repo, item[0], item[1]) result.append(repo) return result def write_services_to_file(): services = [] for repo in list_sub_repos(): configs = repo.parse_pulumi_config() for config in configs: services.extend(config.to_list()) with open(f"{ROOT}parser/meta/services.yaml", "w") as f: f.write(yaml.dump(services)) def write_rpcs_to_file(): rpcs = [] for repo in list_sub_repos(): r = repo.parse_proto() rpcs.extend(r) with open(f"{ROOT}parser/meta/rpcs.yaml", "w") as f: f.write(yaml.dump(rpcs)) def write_swagger_http_to_file(): swagger_https = [] for repo in list_sub_repos(): swagger_https.extend(repo.parse_swagger_http()) with open(f"{ROOT}parser/meta/swagger_https.yaml", "w") as f: f.write(yaml.dump(swagger_https)) def main(): # write_services_to_file() write_rpcs_to_file() write_swagger_http_to_file() main()