""" Dev environment cluster cleanup rules. By default, clusters are deleted after 8 hours past creation times. This can be overridden by specifying "keep-h" (e.g., keep-12h) in the cluster name. The maximum duration is not enforced for now but please keep it under 72h. """ import datetime import logging import model import os import re import requests import tokens import tool from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool CLUSTER_REGEX_PATTERN = r'keep-(\d+)h' DB_ENGINE_URL = "mysql+pymysql://{}:{}@{}:{}/{}" CLUSTER_DELETE_URL = "/api/v1/orgs/{}/projects/{}/clusters/{}" LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" DATE_FORMAT = "%m/%d/%Y %H:%M:%S %p" logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT, datefmt=DATE_FORMAT) LIVE_STATUSES = ['normal'] # Only treats clusters in "normal" state to avoid accidentally causing more problems. def get_all_live_clusters_with_user_info(cfg): passwd = tool.load_auth_from_k8s_secret(cfg.get('auth').get('secret_name'), cfg.get('common').get('namespace'), cfg.get('auth').get('password_secret_key')) engine_url = DB_ENGINE_URL.format(cfg.get('db').get('user'), passwd, cfg.get('db').get('host'), int(cfg.get('db').get('port')), cfg.get('db').get('database')) try: engine = create_engine(engine_url, encoding='utf-8', echo=True, pool_recycle=-1, poolclass=NullPool) session = sessionmaker(engine) results = session().query(model.Cluster). \ join(model.User, model.User.id == model.Cluster.created_by_id). \ add_entity(model.User). \ filter(model.Cluster.status.in_(LIVE_STATUSES)).all() return results finally: try: session().close() except Exception: logging.exception("DB operation failed") def process_cluster_cleanup(cfg): keep_hour = int(cfg.get('common').get('default_keep_hour')) pending_cleanup = [] for cluster, user in get_all_live_clusters_with_user_info(cfg): now = datetime.datetime.now() match = re.search(CLUSTER_REGEX_PATTERN, cluster.name) if match: keep_hour = int(match.group(1)) if now - cluster.created_at > datetime.timedelta(hours=keep_hour): logging.info(f'Cluster {cluster.name}[{cluster.id}] has expired, created_at={cluster.created_at}, keep_hour={keep_hour}') pending_cleanup.append((cluster, user)) else: logging.info(f'Cluster {cluster.name}[{cluster.id}] is still within keep_hour, created_at={cluster.created_at}, keep_hour={keep_hour}') # execute cleanup operations through central cluster api for cluster, user in pending_cleanup: logging.info(f'Deleting cluster {cluster.name}[{cluster.id}]') try: token = tokens.fetch_dbaas_token(cfg, user.email) delete_cluster(cfg, cluster, token) except Exception: logging.exception("Delete Cluster failed, cluster_id=<%s>", cluster.id) logging.info(f'Deletion request sent for cluster {cluster.name}[{cluster.id}]') def delete_cluster(cfg, cluster, token): header = {'Authorization': token} url_prefix = cfg.get('environment').get('central_url_prefix') delete_url = url_prefix + CLUSTER_DELETE_URL.format(cluster.tenant_id, cluster.project_id, cluster.id) requests.delete(delete_url, headers=header) def process(): config_path = os.getenv("config-path") cfg = tool.read_yaml(config_path) process_cluster_cleanup(cfg) if __name__ == '__main__': process()