import time import _thread import o11ydiagnosis.subtasks.slowlogconverter import o11ydiagnosis.subtasks.slowlogcompactor from typing import Dict from o11ydiagnosis.subtasks.slowlogconverter.objstore import S3 clusters = [ { "tenant": "1372813089208821887", "project": "1372813089454312498", "cluster": "1379661944646258403", }, ] def main(): s3 = S3() for cluster in clusters: _thread.start_new_thread(process, (s3, cluster)) while True: time.sleep(600) def process(s3: S3, cluster: Dict[str, str]): converter = o11ydiagnosis.subtasks.slowlogconverter.Task( s3, tenant=cluster["tenant"], cluster=cluster["cluster"], o11y_cluster="xxxx", vendor="aws", region="ap-southeast-1", o11y_bucket="o11y-prod-shared-ap-southeast-1", o11y_prefix=f"0/{cluster['tenant']}/{cluster['project']}/{cluster['cluster']}/", drhouse_bucket="o11y-prod-shared-ap-southeast-1", drhouse_prefix=f"diagnosis/data/{cluster['cluster']}/", drhouse_local_path=f"./data/{cluster['cluster']}/", ) compactor = o11ydiagnosis.subtasks.slowlogcompactor.Task( s3, cluster=cluster["cluster"], drhouse_bucket="o11y-prod-shared-ap-southeast-1", drhouse_prefix=f"diagnosis/data/{cluster['cluster']}/", drhouse_local_path=f"./data/{cluster['cluster']}/", ) # now = int(time.time()) # now -= now % 3600 now = 1740369600 now -= now % 3600 begin = 1740358800 while now >= begin: converter.run(now) converter.upload_modified_objects() for n_hours in [3, 6, 12, 24]: if now % (3600 * n_hours) == 0: begin_hour_ts = now - 3600 * n_hours compactor.prepare(begin_hour_ts, n_hours) compactor.run(begin_hour_ts, n_hours) compactor.upload_modified_objects() now -= 3600 if __name__ == "__main__": main()