import os from o11ydiagnosis.subtasks.slowlogcompactor.log import log from o11ydiagnosis.subtasks.slowlogcompactor.task import Task from o11ydiagnosis.subtasks.slowlogconverter import drhouse_s3_bucket, drhouse_s3_prefix from o11ydiagnosis.subtasks.slowlogconverter.objstore import S3, GCS, AzureBlob class Config: n_hours: int def __init__(self, task, task_info, subtask_info): self.engine_task = task self.local_path = task.cache_prefix provider: str = task_info["meta"]["provider"].lower() # aws/gcp region: str = task_info["meta"]["region"] # TODO: use env var ? self.cluster: str = task_info["meta"]["clusterID"] self.n_hours: int = subtask_info["hours"] assert self.n_hours > 1 if provider == "aws": s3 = S3() elif provider == "gcp": s3 = GCS() elif provider == "azure": s3 = AzureBlob() else: raise ValueError("invalid provider: " + provider) drhouse_bucket = drhouse_s3_bucket(region, "prod") drhouse_prefix = drhouse_s3_prefix(self.cluster) drhouse_local_path = self.local_path.rstrip("/") self.task = Task(s3, self.cluster, drhouse_bucket, drhouse_prefix, drhouse_local_path) def is_timing(self, ts: int) -> bool: # utc_time = datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") # log.info(f"[{self.cluster}] calling is_timing(ts={ts}) (utc time: {utc_time})") return ts % (self.n_hours * 3600) == 600 def ready(self, ts: int) -> bool: # utc_time = datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") # log.info(f"[{self.cluster}] calling ready(ts={ts}) (utc time: {utc_time})") ts -= ts % (self.n_hours * 3600) ts -= self.n_hours * 3600 self.task.prepare(ts, self.n_hours) return len(self.task.existed_statsdb_paths) > 0 def start(self, ts: int): # utc_time = datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") # log.info(f"[{self.cluster}] calling start(ts={ts}) (utc time: {utc_time})") ts -= ts % (self.n_hours * 3600) ts -= self.n_hours * 3600 self.task.run(ts, self.n_hours) def close(self, ts: int): # utc_time = datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d %H:%M:%S") # log.info(f"[{self.cluster}] calling close(ts={ts}) (utc time: {utc_time})") paths = self.task.dirty_merged_statsdb_paths.keys() paths = [path.removeprefix(self.local_path).lstrip("/") for path in paths] if paths: log.info(f"[{self.cluster}] uploading paths: {paths}") self.engine_task.push(paths) for merged_statsdb_path in self.task.dirty_merged_statsdb_paths.keys(): os.remove(merged_statsdb_path) self.task.dirty_merged_statsdb_paths.clear()