import json from typing import Dict, List, Set from collections import defaultdict def build_index(results: List[dict], region_key, name_key) -> Dict[str, dict]: index = {} for item in results: metric = item.get("metric", {}) region = metric.get(region_key) name = metric.get(name_key) if not region or not name: continue uk = f"{region}/{name}" index[uk] = item return index def region_to_gcp_k8s_region(region: str): if 'asia' in region: return 'as' elif 'us' in region: return 'us' elif 'europe' in region: return 'eu' else: print("[WARN] build_network_json: Unknown region pattern for gcp k8s region:", region) return "" def build_vpc_uk2approutes(results: List[dict]) -> Dict[str, Set[str]]: index = defaultdict(set) for item in results: metric = item.get("metric", {}) region = metric.get("region") vpc = metric.get("vpc") approute_name = metric.get("name") if not region or not vpc: continue uk = f"{region}/{vpc}" index[uk].add(approute_name) return dict(index) def build_approute_name2item(results: List[dict]) -> Dict[str, dict]: index = {} for item in results: metric_item = item.get("metric", {}) name = metric_item.get("name") if name: index[name] = metric_item return index def extract_metric(item: dict): return item.get("metric", {}) if item else {} def rebuild_network( vpc_data: dict, internet_data: dict, appbox_data: dict, approute_data: dict, ) -> List[dict]: vpcs = vpc_data["data"]["result"] internets = internet_data["data"]["result"] appboxes = appbox_data["data"]["result"] approutes = approute_data["data"]["result"] # Build indexes and deduplicate data(dev 1,stag 2, prod 3) vpc_by_name = build_index(vpcs, "region", "name") vpcs = list(vpc_by_name.values()) internet_by_vpc_uk = build_index(internets, "region", "vpc") appbox_by_vpc_uk = build_index(appboxes, "region", "vpc") approute_by_vpc_uk_set = build_vpc_uk2approutes(approutes) approutes_by_name = build_approute_name2item(approutes) networks = [] print("[INFO] VPC count after deduplication:", len(vpcs)) print("[INFO] Internet count after deduplication:", len(internet_by_vpc_uk)) print("[INFO] Appbox count after deduplication:", len(appbox_by_vpc_uk)) print("[INFO] Approute count after deduplication:", len(approutes_by_name)) for vpc_item in vpcs: vpc = extract_metric(vpc_item) vpc_uk = f"{vpc.get('region')}/{vpc.get('name')}" internet = extract_metric(internet_by_vpc_uk.get(vpc_uk)) appbox = extract_metric(appbox_by_vpc_uk.get(vpc_uk)) approute_name_set = approute_by_vpc_uk_set.get(vpc_uk, set()) approute_status = {} for approute_name in approute_name_set: approute_item = approutes_by_name.get(approute_name) status = approute_item.get("status", "Unknown") approute_status[approute_name] = status # get env and exported_namespace from prometheus exported_namespace, env = '', '' if 'prometheus' in vpc: env = vpc['prometheus'].replace('monitoring/', '').split('-')[0] if env in ['prod', 'staging'] and 'project_id' in vpc: exported_namespace = env + '-' + vpc['project_id'] if env == 'dev' and 'project_id' in vpc: exported_namespace = 'nightly-ms-' + vpc['project_id'] # deal with dataplane label dataplane = appbox.get("dataplane", "") provider = vpc.get("provider", "") region = vpc.get("region", "") vpc_id = vpc.get("vpc_id", "") name = provider + "-" + region if provider == 'gke' and 'dedicated-' not in dataplane: if 'shoot' in vpc_id: dataplane = vpc_id.split('--')[-1] name = name.replace('gke', 'gcp') else: if env == 'dev': dataplane = vpc_id elif env == 'staging': dataplane = vpc_id.replace(f"-{region}", "") elif env == 'prod': r = region_to_gcp_k8s_region(region) dataplane = vpc_id.replace(f"{region}", r) network = { "metric": { "__name__": "infra_exporter_network", # ---- from vpc ---- "account": vpc.get("account", ""), "cidr": vpc.get("cidr", ""), "project_id": vpc.get("project_id", ""), "tenant_id": vpc.get("tenant_id", ""), "provider": provider, "region": region, "exported_provider": vpc.get("exported_provider", ""), "exported_region": vpc.get("exported_region", ""), "exported_namespace": exported_namespace, "env": env, "vpc": vpc_id, # ---- infra / o11y ---- "cluster": vpc.get("cluster", ""), "control_plane_info": vpc.get("control_plane_info", ""), "o11y_cluster_info": vpc.get("o11y_cluster_info", ""), "o11y_id": vpc.get("o11y_id", ""), "prometheus": vpc.get("prometheus", ""), "instance": vpc.get("instance", ""), "job": "infra-exporter", "service": "infra-exporter", "name": name, "namespace": "infra", "endpoint": "metrics", # ---- from appbox ---- "dataplane": dataplane, "paused": appbox.get("paused", ""), "vpc_status": vpc.get("status", ""), "internet_status": internet.get("status", ""), "appbox_status": appbox.get("status", ""), "approutes": approute_status, }, "value": vpc_item.get("value", ""), } networks.append(network) return networks def dump_to_file(networks: List[dict], path: str): with open(path, "w", encoding="utf-8") as f: json.dump(networks, f, indent=2, ensure_ascii=False)