#### # detect the leaked aws vm instance on dev environments # it will be added into gcloud function later ### import json import subprocess import os import argparse import yaml from kubernetes import client, config import base64 import time import csv from datetime import datetime import boto3 instances = [] instanceDict = {} for region in ["us-west-2", "us-east-1", "ap-northeast-1", "ap-northeast-2", "ap-southeast-1"]: print("fetch ec2 instances from region {}".format(region)) ec2client = boto3.client('ec2', region_name=region) response = ec2client.describe_instances() for reservation in response['Reservations']: for instance in reservation['Instances']: keyName = '' if "KeyName" in instance: keyName = instance['KeyName'] instances.append( [ instance['InstanceId'], instance['InstanceType'], instance["LaunchTime"], keyName, region ], ) instanceDict[instance['InstanceId']] = [ instance['InstanceId'], instance['InstanceType'], instance["LaunchTime"], keyName, region, ], eksClusters = { "dev-base-us-west-2": "us-west-2", "dev-seed-us-west-2": "us-west-2", "stability-base-us-west-2": "us-west-2", "stability-seed-us-west-2": "us-west-2", "staging-base-us-west-2": "us-west-2", "staging-seed-us-west-2": "us-west-2", "dev-seed-us-east-1": "us-east-1", "staging-seed-us-east-1": "us-east-1", "staging-seed-ap-northeast-1": "ap-northeast-1", "staging-seed-ap-northeast-2": "ap-northeast-2", "staging-seed-ap-southeast-1": "ap-southeast-1", } # eksClusters = { # "prod-base-us-west-2": "us-west-2", # "prod-seed-us-west-2": "us-west-2", # "prod-seed-us-east-1": "us-east-1", # "prod-seed-ap-northeast-1": "ap-northeast-1", # "prod-seed-ap-northeast-2": "ap-northeast-2", # "prod-seed-ap-southeast-1": "ap-southeast-1", # } outfile = 'aws-instance.csv' csvFile = open(outfile, 'w', newline='') fieldnames = ['id', 'name', 'LaunchTime', 'InstanceType', 'zone'] writer = csv.DictWriter(csvFile, fieldnames=fieldnames) writer.writeheader() allRunInstances = [] def process_eks_clusters(): for eks in eksClusters: os.system('aws eks update-kubeconfig --name="{}" --region {}'.format(eks, eksClusters[eks])) config.kube_config.load_kube_config() apiClient = client.CoreV1Api() nodes = apiClient.list_node() for node in nodes.items: seps = str.split(node.spec.provider_id, "/") allRunInstances.append(seps[len(seps) -1]) print("process eks {} node {} id {}".format(eks, node.metadata.name, seps[len(seps) -1])) def process_all_shoot_clusters(): for eks in eksClusters: os.system('aws eks update-kubeconfig --name="{}" --region {}'.format(eks, eksClusters[eks])) config.kube_config.load_kube_config() apiClient = client.CoreV1Api() namespaces = apiClient.list_namespace() for ns in namespaces.items: os.system('aws eks update-kubeconfig --name="{}" --region {}'.format(eks, eksClusters[eks])) config.kube_config.load_kube_config() apiClient = client.CoreV1Api() if 'shoot--' not in ns.metadata.name: continue print("process shoot {}".format(ns.metadata.name)) try: shootKubeconfig = apiClient.read_namespaced_secret('kubecfg', ns.metadata.name) shootKubeconfig = shootKubeconfig.data["kubeconfig"] shootKubeconfig= base64.b64decode(shootKubeconfig) shootKubeconfigFile= open(ns.metadata.name, "w") yaml.dump(yaml.load(shootKubeconfig), shootKubeconfigFile) shootKubeconfigFile.close() config.kube_config.load_kube_config(config_file=ns.metadata.name) apiClient = client.CoreV1Api() nodes = apiClient.list_node() namespaces = apiClient.list_namespace() pods = [] for ns in namespaces.items: namespaced_pods = apiClient.list_namespaced_pod(ns.metadata.name) for pod in namespaced_pods.items: pods.append(pod) for node in nodes.items: print("process shoot {} node {}".format(ns.metadata.name, node.metadata.name)) if "worker.gardener.cloud/pool" in node.metadata.labels and\ node.metadata.labels["worker.gardener.cloud/pool"] == "master": seps = str.split(node.spec.provider_id, "/") allRunInstances.append(seps[len(seps) -1 ]) continue if "worker.garden.sapcloud.io/group" in node.metadata.labels and \ node.metadata.labels["worker.garden.sapcloud.io/group"] == "master": seps = str.split(node.spec.provider_id, "/") allRunInstances.append(seps[len(seps) -1 ]) continue nodeFoundIt = False for pod in pods: if 'extra' in pod.metadata.name or \ 'db-pd' in pod.metadata.name or \ 'db-tidb' in pod.metadata.name or \ 'db-tikv' in pod.metadata.name or \ "db-tiflash" in pod.metadata.name: if pod.spec.node_name is not None and pod.spec.node_name == node.metadata.name: nodeFoundIt = True break if nodeFoundIt: seps = str.split(node.spec.provider_id, "/") allRunInstances.append(seps[len(seps) -1 ]) print("process shoot {} node {} id {}".format(ns.metadata.name, node.metadata.name, seps[len(seps) -1])) except Exception as e: print("bypass shoot {}, because {}".format(ns.metadata.name, e)) process_eks_clusters() process_all_shoot_clusters() for instance in allRunInstances: print(instance) for instance in instances: foundIt = False for runInstance in allRunInstances: if instance[0] == runInstance: foundIt = True if not foundIt: writer.writerow( { 'id': instance[0], 'name': instance[3], 'LaunchTime': instance[2], 'InstanceType': instance[1], 'zone': instance[4] } ) csvFile.close()