from os import path, environ import sys pa = path.abspath(path.join(path.dirname(__file__), '../../..')) sys.path.append(pa) from diagnosis.dataloader.s3loader import S3Loader from diagnosis.dataloader.gcsloader import GCSLoader from diagnosis.dataloader.localloader import LocalFileLoader from diagnosis.task.manager import Manager, EngineConfig def main(): # environ['ENGINE_ID'] = 'diag-engine-0' # environ['SKIP_SLOWLOG'] = 'True' # environ['BUCKET_NAME']='o11y-dev-shared-us-west-2' # environ['CACHE_PATH']='cache_path' # environ['RUN_ONCE'] = 'True' # environ['SKIP_PREPROCESS'] = 'True' # environ['FETCH_TIMESTEMP'] = '1682179200' # environ['CHECKPOINT']= '1684800000' #disabled stdout buffer for debug sys.stdout = open(sys.stdout.fileno(), mode='w', buffering=1) # init loader loader = get_loader() engine_config = EngineConfig() # execute the task manager task_manager = Manager(loader, engine_config) task_manager.start() pass def get_loader(): if 'USE_LOCAL_LOADER' in environ and environ['USE_LOCAL_LOADER'] == 'True': return LocalFileLoader(environ['LOCAL_LOADER_PATH']) else: if 'REMOTE_PROVIDER' in environ and environ['REMOTE_PROVIDER'] == 'gcp': return GCSLoader(environ['BUCKET_NAME'], environ['GCP_PROJECT']) return S3Loader(environ['BUCKET_NAME']) if __name__ == '__main__': main()