#### # detect the leaked gcp disk on dev environments # it will be added into gcloud function later ### import json import subprocess import os import argparse import yaml from kubernetes import client, config import base64 import time import csv from datetime import datetime import boto3 disks = [] disksDict = {} for region in ["us-west-2", "us-east-1", "ap-northeast-1", "ap-northeast-2", 'ap-southeast-1']: ec2client = boto3.client('ec2', region_name=region) response = ec2client.describe_volumes() for reservation in response['Volumes']: disks.append( [ reservation['VolumeId'], reservation['Size'], reservation['CreateTime'], reservation['VolumeType'], region ] ) disksDict[reservation['VolumeId']] = [ reservation['VolumeId'], reservation['Size'], reservation['CreateTime'], reservation['VolumeType'], region ] eksClusters = { "dev-base-us-west-2": "us-west-2", "dev-seed-us-west-2": "us-west-2", "stability-base-us-west-2": "us-west-2", "stability-seed-us-west-2": "us-west-2", "staging-base-us-west-2": "us-west-2", "staging-seed-us-west-2": "us-west-2", "dev-seed-us-east-1": "us-east-1", "staging-seed-us-east-1": "us-east-1", "staging-seed-ap-northeast-1": "ap-northeast-1", "staging-seed-ap-northeast-2": "ap-northeast-2", "staging-seed-ap-southeast-1": "ap-southeast-1", } outfile='aws-disks.csv' csvFile = open(outfile, 'w', newline='') fieldnames = ['id', 'size', 'CreateTime', "VolumeType", "zone"] writer = csv.DictWriter(csvFile, fieldnames=fieldnames) writer.writeheader() allPVs = [] def process_eks_clusters(): for eks in eksClusters: os.system('aws eks update-kubeconfig --name="{}" --region {}'.format(eks, eksClusters[eks])) config.kube_config.load_kube_config() apiClient = client.CoreV1Api() pvs = apiClient.list_persistent_volume() for pv in pvs.items: if pv.spec.aws_elastic_block_store is not None: print("process eks {} pv {}".format(eks, pv.metadata.name)) seps = str.split(pv.spec.aws_elastic_block_store.volume_id, "/") allPVs.append(seps[len(seps) -1 ]) def process_all_shoot_clusters(): for eks in eksClusters: os.system('aws eks update-kubeconfig --name="{}" --region {}'.format(eks, eksClusters[eks])) config.kube_config.load_kube_config() apiClient = client.CoreV1Api() namespaces = apiClient.list_namespace() for ns in namespaces.items: os.system('aws eks update-kubeconfig --name="{}" --region {}'.format(eks, eksClusters[eks])) config.kube_config.load_kube_config() apiClient = client.CoreV1Api() if 'shoot--' not in ns.metadata.name: continue print("process shoot {}".format(ns.metadata.name)) try: shootKubeconfig = apiClient.read_namespaced_secret('kubecfg', ns.metadata.name) shootKubeconfig = shootKubeconfig.data["kubeconfig"] shootKubeconfig= base64.b64decode(shootKubeconfig) shootKubeconfigFile= open(ns.metadata.name , "w") yaml.dump(yaml.load(shootKubeconfig), shootKubeconfigFile) shootKubeconfigFile.close() config.kube_config.load_kube_config(config_file=ns.metadata.name) apiClient = client.CoreV1Api() pvs = apiClient.list_persistent_volume() for pv in pvs.items: if pv.spec.aws_elastic_block_store is not None: print("process eks {} pv {}".format(eks, pv.metadata.name)) seps = str.split(pv.spec.aws_elastic_block_store.volume_id, "/") allPVs.append(seps[len(seps) -1 ]) except: print("bypass shoot {}".format(ns.metadata.name)) process_eks_clusters() process_all_shoot_clusters() for instance in allPVs: print(instance) for disk in disksDict: if disk not in set(allPVs): writer.writerow( { 'id': disksDict[disk][0], 'size': disksDict[disk][1], 'CreateTime': disksDict[disk][2], "VolumeType": disksDict[disk][3], "zone": disksDict[disk][4], } ) csvFile.close()