import os import time import boto3 from kubernetes import client, config import pymysql from prometheus_client import Gauge, start_http_server aws_gp2 = Gauge('aws_gp2_total_size', 'aws gp2 total size') aws_gp3 = Gauge('aws_gp3_total_size', 'aws gp3 total size') aws_awailable_gp2 = Gauge('aws_gp2_avail_size', 'aws gp2 awailable size') aws_awailable_gp3 = Gauge('aws_gp3_avail_size', 'aws gp3 awailable size') vm_total_cpus = Gauge('aws_ec2_total_cpus', 'aws cpus total size') vm_total_memorys = Gauge('aws_ec2_total_memorys', 'aws memory total size') aws_tidb_total_cpus = Gauge('aws_tidb_total_cpus', 'aws cpus total size') aws_tidb_total_memorys = Gauge('aws_tidb_total_memorys', 'aws memory total size') shoot_nat_numbers = Gauge('shoot_nat_number', 'shoot nat total size') aws_nat_numbers = Gauge('aws_nat_number', 'aws nat total size') aws_shoots = Gauge('aws_shoots', 'aws shoots') aws_control_plane_cpus = Gauge('aws_control_plane_cpus', 'aws control plane cpu total size') aws_control_plane_memorys = Gauge('aws_control_plane_memorys', 'aws contral plan memory total size') aws_others_cpus = Gauge('aws_others_cpus', 'aws others cpu total size') aws_others_memorys = Gauge('aws_others_memorys', 'aws others[ memory total size') shoot_idle_cpus = Gauge('shoot_idle_cpus', 'shoot idle cpu total size') shoot_idle_memorys = Gauge('shoot_idle_memorys', 'shoot idle memory total size') tidb_clusters = Gauge('tidb_clusters', 'tidb clusters numbers') free_tier_clusters = Gauge('free_tier_tidb_clusters', 'free tier clusters numbers') nlb_numbers = Gauge('aws_nlb_numbers', 'aws nlb numbers') shoot_nlb_numbers = Gauge('aws_shoot_nlb_numbers', 'aws shoot nlb numbers') aws_instances_profiles = {} def compute_nlb(): shoot_nlbs = 0 nlbs = 0 for region in ["us-west-2", "us-east-1", "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "eu-central-1"]: client = boto3.client('elbv2', region_name=region) response = client.describe_load_balancers() for nlb in response["LoadBalancers"]: nlbArn = nlb['LoadBalancerArn'] response = client.describe_tags( ResourceArns=[ nlbArn, ] ) tags = response['TagDescriptions'][0]['Tags'] name = "" for tag in tags: if tag['Key'] == 'KubernetesCluster': name = tag['Value'] if 'shoot' in name: shoot_nlbs = shoot_nlbs + 1 nlbs = nlbs + 1 client = boto3.client('elb', region_name=region) response = client.describe_load_balancers() for nlb in response["LoadBalancerDescriptions"]: nlbArn = nlb['LoadBalancerName'] response = client.describe_tags( LoadBalancerNames=[ nlbArn, ] ) tags = response['TagDescriptions'][0]['Tags'] name = "" for tag in tags: if tag['Key'] == 'KubernetesCluster': name = tag['Value'] if 'shoot' in name: shoot_nlbs = shoot_nlbs + 1 nlbs = nlbs + 1 nlb_numbers.set(nlbs) shoot_nlb_numbers.set(shoot_nlbs) print("total nlbs", nlbs) print("shoot nlbs", shoot_nlbs) def compute_aws_volume(): gp2_total_size = 0 gp3_total_size = 0 gp2_avail_size = 0 gp3_avail_size = 0 for region in ["us-west-2", "us-east-1", "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "eu-central-1"]: ec2client = boto3.client('ec2', region_name=region) volumes = ec2client.describe_volumes() for volume in volumes['Volumes']: if volume['State'] == "available": if volume['VolumeType'] == "gp2": gp2_avail_size = gp2_avail_size + volume['Size'] elif volume['VolumeType'] == "gp3": gp3_avail_size = gp3_avail_size + volume['Size'] else: if volume['VolumeType'] == "gp2": gp2_total_size = gp2_total_size + volume['Size'] elif volume['VolumeType'] == "gp3": gp3_total_size = gp3_total_size + volume['Size'] print("gp2_total_size", gp2_total_size) print("gp3_total_size", gp2_total_size) print("gp3_avail_size", gp3_avail_size) print("gp2_avail_size", gp2_avail_size) aws_gp2.set(gp2_total_size) aws_gp3.set(gp3_total_size) aws_awailable_gp3.set(gp3_avail_size) aws_awailable_gp2.set(gp2_avail_size) def compute_aws_cpus_and_memorys(): cpus = 0 memorys = 0 tidb_cpu = 0 tidb_memorys = 0 idle_shoot_cpu = 0 idle_shoot_memory = 0 controlplane_cpu = 0 controlplane_memory = 0 other_cpu = 0 other_memory = 0 for region in ["us-west-2", "us-east-1", "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "eu-central-1"]: ec2client = boto3.client('ec2', region_name=region) filters = [ { 'Name': 'instance-state-name', 'Values': ['running'] } ] response = ec2client.describe_instances(Filters=filters) for reservation in response['Reservations']: for instance in reservation['Instances']: instance_type_name = instance['InstanceType'] if instance_type_name not in aws_instances_profiles: instance_type = ec2client.describe_instance_types(InstanceTypes=[instance_type_name]) aws_instances_profiles[instance_type_name] = instance_type # print(instance) current_vcpu_count = aws_instances_profiles[instance_type_name]['InstanceTypes'][0]['VCpuInfo']['DefaultVCpus'] cpus = cpus + current_vcpu_count current_vmemory_count = aws_instances_profiles[instance_type_name]['InstanceTypes'][0]['MemoryInfo']['SizeInMiB'] memorys = memorys + current_vmemory_count/1024 if 'Tags' not in instance: continue tags = instance['Tags'] name = "" for tag in tags: if tag['Key'] == 'Name': name = tag['Value'] if "shoot" in name and 'master' not in name: tidb_cpu = tidb_cpu + current_vcpu_count tidb_memorys = tidb_memorys + current_vmemory_count/1024 elif "shoot" in name and 'master' in name: idle_shoot_cpu = idle_shoot_cpu + current_vcpu_count idle_shoot_memory = idle_shoot_memory + current_vmemory_count/1024 elif "dev-seed" in name or "staging-seed" in name: controlplane_cpu = controlplane_cpu + current_vcpu_count controlplane_memory = controlplane_memory + current_vmemory_count/1024 elif "dev-base" in name or "staging-base" in name: controlplane_cpu = controlplane_cpu + current_vcpu_count controlplane_memory = controlplane_memory + current_vmemory_count/1024 else: if 'aws:autoscaling:groupName' in tags and ['base', "seed"] in tags['aws:autoscaling:groupName']: controlplane_cpu = controlplane_cpu + current_vcpu_count controlplane_memory = controlplane_memory + current_vmemory_count/1024 else: other_cpu = other_cpu + current_vcpu_count other_memory = other_memory + current_vmemory_count/1024 vm_total_cpus.set(cpus) vm_total_memorys.set(memorys) aws_tidb_total_cpus.set(tidb_cpu) aws_tidb_total_memorys.set(tidb_memorys) shoot_idle_cpus.set(idle_shoot_cpu) shoot_idle_memorys.set(idle_shoot_memory) aws_control_plane_cpus.set(controlplane_cpu) aws_control_plane_memorys.set(controlplane_memory) aws_others_cpus.set(other_cpu) aws_others_memorys.set(other_memory) print("cpu: ", cpus, "memory: ", memorys) print("tidb cpu: ", tidb_cpu, "tidb memory: ", tidb_memorys) def compute_shoots(): aws_shoot_counts = 0 config.kube_config.load_kube_config(config_file="gardener-dev") shootClient = client.CustomObjectsApi() nightlyShoots = shootClient.list_namespaced_custom_object( group="core.gardener.cloud", version="v1beta1", namespace="garden-nightly", plural="shoots", ) config.kube_config.load_kube_config(config_file="gardener-staging") shootClient = client.CustomObjectsApi() stagingShoots = shootClient.list_namespaced_custom_object( group="core.gardener.cloud", version="v1beta1", namespace="garden-staging", plural="shoots", ) for shoot in nightlyShoots['items']: if shoot['spec']['provider']['type'] == 'aws': aws_shoot_counts = aws_shoot_counts + 1 for shoot in stagingShoots['items']: if shoot['spec']['provider']['type'] == 'aws': aws_shoot_counts = aws_shoot_counts + 1 aws_shoots.set(aws_shoot_counts) print("aws shoot counts: ", aws_shoot_counts) def compute_tidbclusters(): clusters_number = 0 dev_clusters_number = 0 # password = { # 'jiajun': ":iM&9n84@x#AK{U6aQZe", # 'liyilong2': "Y5CYl%mYmXUM_:yIGpHC_m9)A_biSUIU@=n', # "chenfei": "?elLX&8VEmi]P5[Q?fpf", # "infra-dev": "PhK:rAh@bPxyYwT{9)C0", # "jiahao3": "o!A0kVW{SX5[Vy{yc%DC", # "jiang": '2OrY2w+UPP%[j&DvD]q$', # 'liusx': ')[&43vrUg}KQGw$MwzV0', # } # for user in ['jiajun', 'liyilong2', # 'yinliang3', 'louis', # "chenfei", "infra-dev" # , "jiahao3", "jiang", 'liusx', ]: # passwd = password[user] # db = pymysql.connect(host="db-tidb." + user, # port=4000, # user='root', # password=passwd, # db= 'dbaas', charset='utf8') # cursor = db.cursor() # cursor.execute("select * from clusters where status='normal' and cluster_profile_id not in (select id from cluster_profiles where name like '%dev');") # results = cursor.fetchall() # clusters_number = clusters_number + len(results) # cursor.execute("select * from clusters where status='normal' and cluster_profile_id in (select id from cluster_profiles where name like '%dev');") # results = cursor.fetchall() # dev_clusters_number = dev_clusters_number + len(results) for user in ['dev', 'staging', 'hanlins', 'yinliang3']: passwd = os.getenv('dbpassword') db = pymysql.connect(host="tidb.e23bdccc.cb17aaf2.us-west-2.prod.aws.tidbcloud.com", port=4000, user='root', password=passwd, db= user+'_central', charset='utf8') cursor = db.cursor() cursor.execute("select * from clusters where status='normal' and cluster_profile_id not in (select id from cluster_profiles where name like '%dev');") results = cursor.fetchall() clusters_number = clusters_number + len(results) cursor.execute("select * from clusters where status='normal' and cluster_profile_id in (select id from cluster_profiles where name like '%dev');") results = cursor.fetchall() dev_clusters_number = dev_clusters_number + len(results) tidb_clusters.set(clusters_number) free_tier_clusters.set(dev_clusters_number) print("tidb cluster numbers: ", clusters_number) print("dev tidb cluster numbers: ", dev_clusters_number) # def compute_tidbclusters(): # tidbclusters = 0 # dev_tidbclusters = 0 # # config.load_kube_config(config_file="infra-dev-kubeconfig") # v1 = client.CoreV1Api() # namespaces = v1.list_namespace() # tidbClient = client.CustomObjectsApi() # for ns in namespaces.items: # clusters = tidbClient.list_namespaced_custom_object( # group="cloud.pingcap.com", # version="v1", # namespace=ns.metadata.name, # plural="clusters", # ) # for cluster in clusters['items']: # if cluster['status']['phase'] == 'Running': # if 'free-tier' in cluster['spec']['networkName']: # dev_tidbclusters = dev_tidbclusters + 1 # else: # tidbclusters = tidbclusters + 1 # # config.load_kube_config(config_file="infra-staging-kubeconfig") # v1 = client.CoreV1Api() # namespaces = v1.list_namespace() # tidbClient = client.CustomObjectsApi() # for ns in namespaces.items: # clusters = tidbClient.list_namespaced_custom_object( # group="cloud.pingcap.com", # version="v1", # namespace=ns.metadata.name, # plural="clusters", # ) # for cluster in clusters['items']: # if cluster['status']['phase'] == 'Running': # if 'free-tier' in cluster['spec']['networkName']: # dev_tidbclusters = dev_tidbclusters + 1 # else: # tidbclusters = tidbclusters + 1 # # tidb_clusters.set(tidbclusters) # free_tier_clusters.set(dev_tidbclusters) def compute_nats(): aws_nats = 0 shoot_nats = 0 for region in ["us-west-2", "us-east-1", "ap-northeast-1", "ap-northeast-2", "ap-southeast-1", "eu-central-1"]: ec2client = boto3.client('ec2', region_name=region) response = ec2client.describe_nat_gateways() for nat in response["NatGateways"]: tags = nat['Tags'] name = "" for tag in tags: if tag['Key'] == 'Name': name = tag['Value'] if "shoot" in name: shoot_nats = shoot_nats + 1 aws_nats = aws_nats + 1 shoot_nat_numbers.set(shoot_nats) aws_nat_numbers.set(aws_nats) print("shoot_nats", shoot_nats) print("aws_nats", aws_nats) compute_aws_volume() compute_aws_cpus_and_memorys() compute_nats() compute_tidbclusters() compute_shoots() compute_nlb() if __name__ == "__main__": start_http_server(8000) while True: compute_aws_volume() compute_aws_cpus_and_memorys() compute_nats() compute_tidbclusters() compute_shoots() compute_nlb() # time.sleep(30)