import time from os import path, environ import os import sys import _thread pa = path.abspath(path.join(path.dirname(__file__), '../..')) sys.path.append(pa) from datasdk.config import config from datasdk.sources import slowlog from diagnosis.dataloader.s3loader import S3Loader from diagnosis.dataloader.gcsloader import GCSLoader from diagnosis.dataloader.localloader import LocalFileLoader from slowquery.manager import Config,Manager from diagnosis.preprocess.utils.utils import LIB_PATH def main(): #disabled stdout buffer for debug sys.stdout = open(sys.stdout.fileno(), mode='w', buffering=1) cache_prefix = os.environ["CACHE_PATH"] clusters = os.environ["CLUSTER_ID"] cluster_list = clusters.split(";") projects = os.environ["PROJECT_ID"] project_list = projects.split(";") for clusterid, projectid in zip(cluster_list, project_list): cache_path = os.path.join(cache_prefix, clusterid) if not os.path.exists(cache_path): os.makedirs(cache_path) engine_config = Config(cluster_id=clusterid, project_id=projectid, cache_path=cache_path) loader = get_loader(engine_config) # set cgo so file path set_so_path(engine_config) # execute the task manager task_manager = Manager(loader, engine_config) print("{} task_manager.start()".format(task_manager.config.cluster_id)) _thread.start_new_thread(startManager, (task_manager,)) while True: time.sleep(3600) pass def startManager(task_manager): task_manager.start() def get_loader(cfg: Config): if cfg.local_dev: return LocalFileLoader(cfg.local_load_path) else: if cfg.provider == Config.PROVIDER_GCP: return GCSLoader(cfg.bucket_name, cfg.gcp_project) return S3Loader(cfg.bucket_name) def set_so_path(cfg: Config): global LIB_PATH if cfg.local_dev: LIB_PATH = environ["CGO_SO_PATH"] if "CGO_SO_PATH" in environ else "planedecode/lib/decodec_mac_arm64.so" if __name__ == '__main__': # environ['O11Y_AUTH0_CLIENT_ID']='Sd4aekPCa5nMUSQBIvf8eGSL895WtV4o' # environ['O11Y_AUTH0_CLIENT_SECRET']='-SLP5UvxJdwGbVaKupxgv53T6otXguApPoG0fBsWCIqDG8_TuvlYnlkhPQHcWaKT' # environ['CLUSTER_ID']='1379661944646413136;1379661944646413137;1379661944646413139;1379661944646413143;1379661944646413150;1379661944646413225;1379661944646413411;1379661944646413610' # environ['PROJECT_ID']='1372813089454523548;1372813089454523725;1372813089454521758;1372813089454525346;1372813089454521730;1372813089454528367;1372813089454530656;1372813089454521730' # environ['TENANT_ID']='1372813089209061633' # environ['CACHE_PATH']='./cache' # environ['LOCAL_DEV']='True' # environ['START_TIMESTEMP']='1705017600' # # environ['END_TIMESTEMP']='1703811600' # environ['PROVIDER']='aws' # environ['LOCAL_LOAD_PATH']='./local' # environ['BUCKET_NAME']='o11y-dev-shared-us-west-2' # environ['AWS_PROFILE']='oncall' # environ["CGO_SO_PATH"]='./planedecode/lib/decodec_mac_x64.so' main()