import json import os import requests import sys from collector.build_network_json import rebuild_network, dump_to_file def dump_data(sources): for source in sources: result = get_prom_result(source['url']) dump_prom_results(result, source['filename']) def get_prom_result(url): try: response = requests.get(url, timeout=5) if response.status_code == 200: print('success: {}'.format(url)) return response.json()['data']['result'] else: print('Error: {}, url: {}'.format(response.status_code, url)) return [] except Exception as e: print('Error: {}'.format(e)) return [] def dump_prom_results(result, filename): if os.path.isfile(filename): result += json.load(open(filename, 'r'))['data']['result'] with open(filename, 'w') as f: json.dump({'data': {'result': result}}, f) ENV2ENDPOINTS = { 'dev': { 'base': 'https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/39190b22-bb48-4167-a485-eec9917ac85a/api/v1/query?query={metrics}', 'regional-aws': 'https://www.ds.{region}.aws.observability.tidbcloud.com/internal/metrics/d38bf7a2-afe1-5ce3-b590-7681d0c8cd79/api/v1/query?query={metrics}', 'regional-gcp': 'https://www.ds.{region}.gcp.observability.tidbcloud.com/internal/metrics/d38bf7a2-afe1-5ce3-b590-7681d0c8cd79/api/v1/query?query={metrics}', }, 'staging': { 'base': 'https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/ea5eef3e-2c03-462d-9594-8bdc2299168c/api/v1/query?query={metrics}', 'regional-aws': 'https://www.ds.{region}.aws.observability.tidbcloud.com/internal/metrics/ba7bffa6-679b-6641-65f7-adcb81edbd33/api/v1/query?query={metrics}', 'regional-gcp': 'https://www.ds.{region}.gcp.observability.tidbcloud.com/internal/metrics/ba7bffa6-679b-6641-65f7-adcb81edbd33/api/v1/query?query={metrics}' }, 'prod': { 'base': 'https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/90c5d4fa-ba99-4c40-8b5f-041f6dd28501/api/v1/query?query={metrics}', 'regional-aws': 'https://www.ds.{region}.aws.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820/api/v1/query?query={metrics}', 'regional-gcp': 'https://www.ds.{region}.gcp.observability.tidbcloud.com/internal/metrics/d5d1a915-1d37-22a7-82b8-8cb67cc57820/api/v1/query?query={metrics}', } } ENV2REGIONS = { 'dev': { 'aws': ['us-west-2', 'us-east-1'], 'gcp': ['us-west1', 'us-west2'], }, 'staging': { 'aws': ['us-west-2', 'us-east-1'], 'gcp': ['us-central1', 'us-west1'], }, 'prod': { 'aws': ['us-west-2', 'us-east-1', 'eu-central-1', 'ap-northeast-1', 'ap-northeast-2', 'ap-south-1', 'ap-southeast-1', 'ap-east-1', 'ap-northeast-3'], 'gcp': ['us-west1', 'us-central1', 'us-east4', 'asia-east1', 'asia-northeast1', 'asia-northeast2', 'asia-southeast1', 'asia-east2', 'asia-south1', 'europe-west3'], } } # for region：ap-southeast-3-def (prod)、us-west-2-lyc (staging, prod) # some new regions need specific cluster_id such us 'dff88c5d-a9d8-4123-aff4-95db222fa873' in query url SPECIAL_INFRA_ENDPOINTS = [ 'https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/dff88c5d-a9d8-4123-aff4-95db222fa873/api/v1/query?query={metrics}', 'https://www.ds.us-west-2.aws.observability.tidbcloud.com/internal/metrics/868d86b7-993b-4794-995e-8bf0e2eae180/api/v1/query?query={metrics}', 'https://www.ds.ap-southeast-3.aws.observability.tidbcloud.com/internal/metrics/f6d08304-b62b-4d2f-bd4c-d53845b16b99/api/v1/query?query={metrics}', 'https://www.ds.af-south-1.aws.observability.tidbcloud.com/internal/metrics/019afccb-64fc-7559-93b8-974534a04187/api/v1/query?query={metrics}', 'https://www.ds.ap-northeast-3.aws.observability.tidbcloud.com/internal/metrics/057e7442-b638-4e16-b712-7495e52db69a/api/v1/query?query={metrics}', 'https://www.ds.ap-southeast-2.aws.observability.tidbcloud.com/internal/metrics/20013902-095e-4be7-9df2-75a9905fb81d/api/v1/query?query={metrics}', 'https://www.ds.ap-southeast-5.aws.observability.tidbcloud.com/internal/metrics/cbe023d8-919f-4d86-aea2-cf2aa4a16903/api/v1/query?query={metrics}', 'https://www.ds.ap-southeast-7.aws.observability.tidbcloud.com/internal/metrics/bb680b30-987b-4fef-83ad-0eb6284f32f1/api/v1/query?query={metrics}', 'https://www.ds.eu-west-1.aws.observability.tidbcloud.com/internal/metrics/5ccad509-1d51-4874-a61d-0515c4e3de63/api/v1/query?query={metrics}', 'https://www.ds.sa-east-1.aws.observability.tidbcloud.com/internal/metrics/4996c648-7115-4e33-8d6b-7a0d6acd6ccf/api/v1/query?query={metrics}', 'https://www.ds.us-east-2.aws.observability.tidbcloud.com/internal/metrics/8437501c-a231-42ff-bead-0ffb0a04b6b3/api/v1/query?query={metrics}', ] # INFRA_METRICS_LIST is a list about cluster-controller CRs, including both gardener arch and cloud-native arch INFRA_METRICS_LIST = [ # 3-layer metrics 'regional_server_vpc_info','regional_server_internet_info','regional_server_appbox_info', 'regional_server_approute_info', 'infra_exporter_cluster' ] # BASIC_METRICS_LIST is a list of basic metrics about k8s core objects BASIC_METRICS_LIST = [ 'kube_persistentvolumeclaim_info', 'kube_pod_info', 'kube_node_info', 'kube_pod_labels', 'kube_pod_container_resource_requests', 'kube_node_status_allocatable' ] def collect_metrics_from_base(env: str, metrics_list: list[str]) -> list[dict]: tasks = [] endpoint = ENV2ENDPOINTS[env]['base'] for metrics in metrics_list: tasks.append({ 'url': endpoint.format(metrics=metrics), 'filename': f'{metrics}_asset_snapshot.json' }) return tasks def collect_metrics_from_regional(env: str, provider: str, region: str, metrics_list: list[str]): tasks = [] endpoint = ENV2ENDPOINTS[env][f'regional-{provider}'] for metrics in metrics_list: tasks.append({ 'url': endpoint.format(region=region, metrics=metrics), 'filename': f'{metrics}_asset_snapshot.json' }) return tasks def collect_metrics_from_special_endpoints(metrics_list: list[str]) -> list[dict]: tasks = [] for endpoint in SPECIAL_INFRA_ENDPOINTS: for metrics in metrics_list: tasks.append({ 'url': endpoint.format(metrics=metrics), 'filename': f'{metrics}_asset_snapshot.json' }) return tasks def collect_all(envs: list[str]): base_metrics_list = [] base_metrics_list.extend(BASIC_METRICS_LIST) base_metrics_list.extend(INFRA_METRICS_LIST) regional_metrics_list = [] regional_metrics_list.extend(BASIC_METRICS_LIST) regional_metrics_list.extend(INFRA_METRICS_LIST) tasks = [] for env in envs: tasks.extend(collect_metrics_from_base(env, base_metrics_list)) for provider, regions in ENV2REGIONS[env].items(): for region in regions: tasks.extend(collect_metrics_from_regional(env, provider, region, regional_metrics_list)) # Add tasks for special endpoints tasks.extend(collect_metrics_from_special_endpoints(INFRA_METRICS_LIST)) tasks.extend(collect_metrics_from_special_endpoints(BASIC_METRICS_LIST)) dump_data(tasks) if __name__ == "__main__": envs = sys.argv[1] collect_all(envs.split(',')) networks = rebuild_network( json.load(open("regional_server_vpc_info_asset_snapshot.json")), json.load(open("regional_server_internet_info_asset_snapshot.json")), json.load(open("regional_server_appbox_info_asset_snapshot.json")), json.load(open("regional_server_approute_info_asset_snapshot.json")), ) dump_to_file(networks, "networks.json")