import time import _thread import o11ydiagnosis.subtasks.slowlogconverter import o11ydiagnosis.subtasks.slowlogcompactor from typing import Dict from o11ydiagnosis.subtasks.slowlogconverter.objstore import S3 clusters = [ { "tenant": "1372813089209061633", "project": "1372813089454525346", "cluster": "1379661944646413143", }, { "tenant": "1372813089209061633", "project": "1372813089454525346", "cluster": "1379661944646413655", }, { "tenant": "1372813089209061633", "project": "1372813089454525346", "cluster": "1379661944646413999", }, ] def main(): s3 = S3() for cluster in clusters: _thread.start_new_thread(process, (s3, cluster)) while True: time.sleep(600) def process(s3: S3, cluster: Dict[str, str]): converter = o11ydiagnosis.subtasks.slowlogconverter.Task( s3, tenant=cluster["tenant"], cluster=cluster["cluster"], o11y_cluster="xxxx", vendor="aws", region="us-east-1", o11y_bucket="o11y-prod-shared-us-east-1", o11y_prefix=f"0/{cluster['tenant']}/{cluster['project']}/{cluster['cluster']}/", drhouse_bucket="o11y-prod-shared-us-east-1", drhouse_prefix=f"diagnosis/data/{cluster['cluster']}/", drhouse_local_path=f"./data/{cluster['cluster']}/", ) compactor = o11ydiagnosis.subtasks.slowlogcompactor.Task( s3, cluster=cluster["cluster"], drhouse_bucket="o11y-prod-shared-us-east-1", drhouse_prefix=f"diagnosis/data/{cluster['cluster']}/", drhouse_local_path=f"./data/{cluster['cluster']}/", ) ts = int(time.time()) ts -= ts % 3600 while True: if ts > int(time.time()): print("sleep 60") time.sleep(60) continue hour_ts = ts - 301 hour_ts -= hour_ts % 3600 converter.run(hour_ts) converter.upload_modified_objects() for n_hours in [3, 6, 12, 24]: if ts % (3600 * n_hours) == 600: begin_hour_ts = ts - 3600 * n_hours - 600 compactor.prepare(begin_hour_ts, n_hours) compactor.run(begin_hour_ts, n_hours) compactor.upload_modified_objects() ts += 300 if __name__ == "__main__": main()