import sys import time from datetime import datetime import uuid import os import json from typing import Dict from o11ydiagnosis.subtasks.slowlogconverter.objstore import ObjectStore from stats import process_stats from output import print_json_table,print_message,print_task_info interval = 300 class Task: task_id: str s3: ObjectStore cluster: str compare_ts: int start_ts: int levels: [] drhouse_bucket: str drhouse_prefix: str drhouse_local_path: str log_paths: Dict[str, str] all_statistic_file_paths: str all_statistic_data: Dict[str, list] def __init__(self, s3: ObjectStore, cluster: str, compare_ts: int, start_ts: int, levels: [],drhouse_bucket: str, drhouse_prefix: str, drhouse_local_path: str): self.task_id = uuid.uuid1() self.s3 = s3 self.cluster = cluster self.compare_ts = compare_ts self.start_ts = start_ts self.levels = levels self.drhouse_bucket = drhouse_bucket self.drhouse_prefix = drhouse_prefix.rstrip("/") self.drhouse_local_path = drhouse_local_path.rstrip("/") self.compare_log_paths = {} self.new_log_paths = {} self.all_statistic_file_paths = f"{self.drhouse_local_path}/result-data/{self.task_id}.json" self.all_statistic_data = {} print_task_info(f"the log statistic result will be saved in {self.all_statistic_file_paths}") def prepare(self): ts = self.compare_ts while ts < self.start_ts: file_key = datetime.utcfromtimestamp(ts).strftime("%Y%m%d%H%M") s3_key = f"{self.drhouse_prefix}/log-stats/5m/{file_key}.db" local_file = f"{self.drhouse_local_path}/log-stats/5m/{file_key}.db" existed = self.download(local_file, s3_key) if existed: print_message(f"download log file {s3_key}") else : print_message(f"log file {s3_key} does not exists") sys.exit() data = process_stats(local_file) self.all_statistic_data[file_key] = data ts = ts + interval self.dump_statistic() def append(self): ts = self.start_ts while True: file_key = datetime.utcfromtimestamp(ts).strftime("%Y%m%d%H%M") s3_key = f"{self.drhouse_prefix}/log-stats/5m/{file_key}.db" local_file = f"{self.drhouse_local_path}/log-stats/5m/{file_key}.db" existed = self.download(local_file, s3_key) if existed: print_message(f"download log file {s3_key}, done.") data = process_stats(local_file) self.all_statistic_data[file_key] = data self.dump_statistic() ts = ts + interval else: print_message(f"downloading log file {s3_key}, doing...") continue time.sleep(60) def dump_statistic(self): if not os.path.exists(self.all_statistic_file_paths): os.makedirs(os.path.dirname(self.all_statistic_file_paths), exist_ok=True) with open(self.all_statistic_file_paths,'w',encoding='utf-8') as f: json.dump(self.all_statistic_data, f,ensure_ascii=False) print_json_table(self.all_statistic_data, self.levels) def download(self, local_path, s3_key): if not os.path.exists(local_path): os.makedirs(os.path.dirname(local_path), exist_ok=True) if self.s3.download(self.drhouse_bucket, s3_key, local_path): self.compare_log_paths[local_path] = s3_key return True else: return False else: self.compare_log_paths[local_path] = s3_key return True