import sched import time import boto3 import random from datasdk.sources import meta from datasdk.utils.coresvc import core_pb2 from google.protobuf.json_format import MessageToJson from log import logger PROVIDER_AWS = "aws" class Collector: def __init__(self, conf): self.conf = conf def start(self): sc = sched.scheduler(time.time, time.sleep) random_delay = random.randint(0, 300) logger.info("random delay {}s".format(random_delay)) sc.enter(random_delay, 1, self.run, (sc,)) sc.run() def run(self, sc): sc.enter(self.conf.collector.interval, 1, self.run, (sc,)) self.collect() def collect(self): logger.info("start collect cluster info") all_clusters = [] try: all_clusters = meta.Meta(self.conf.coresvc.address).list_cluster_meta_by_filter( filters=core_pb2.ClusterFilter( cloud_provider_names=[self.conf.collector.provider], is_dev_tier=False, deleted=False, ) ) except Exception as e: logger.error("list cluster info failed: {}".format(e)) return else: s3_cli = boto3.client('s3', region_name=self.conf.collector.region) for cluster in all_clusters.values(): if cluster.region_name != self.conf.collector.region: continue if self.conf.collector.provider == PROVIDER_AWS: self.sync_to_s3(cluster, s3_cli) logger.info("collect cluster info success") def sync_to_s3(self, cluster, s3_cli): object_key = self.conf.s3.prefix + "/" + str(cluster.id) + ".json" data = MessageToJson(cluster) try: s3_cli.put_object(Bucket=self.conf.s3.bucket, Key=object_key, Body=data) except Exception as e: logger.error("sync cluster {} to s3 failed: {}".format(cluster.id, e)) else: logger.info("sync cluster {} to s3 file {} success".format(cluster.id, object_key))