import sys from os import path import os import time import unittest from unittest.mock import patch, MagicMock pa = path.abspath(path.join(path.dirname(__file__), '../..')) sys.path.append(pa) from diagnosis.dataloader.localloader import LocalFileLoader from diagnosis.dataloader.s3loader import S3Loader from diagnosis.task.preprocess.task import DefaultTask from diagnosis.task.preprocess.config import TaskConfig from diagnosis.utils.utils import get_report_prefix # Check if plotly is available try: import plotly.graph_objs as go from diagnosis.task.diagnosis.report_v2 import ReportV2 PLOTLY_AVAILABLE = True except ImportError: PLOTLY_AVAILABLE = False ReportV2 = None class TestDefaultTask(unittest.TestCase): auth0_id = "-" auth0_sec = "-" clinic_token = '-' @unittest.skipUnless(os.environ.get("TEST_DIAGNOSIS_TASK"), "Diagnosis task test requires environment setup") def test_default_task(self): # cluster = os.environ['CLUSTER_ID'] # 1379661944629574082 # start_day = int(os.environ['START_DAY_TIME']) # 1685980800 # day_length = int(os.environ['DAY_LENGTH']) #14 # bucket_name = os.environ['BUCKET_NAME'] # drhouse-slowlog-tmp-shoutheast-1 # loader = S3Loader(bucket_name) os.environ['AWS_PROFILE'] = 'ob-prod-oncall' cluster = '1379661944636804070' start_day = 1687536000 i = 0 while i < 14: start = start_day + i*24*60*60 end = start + 24*60*60 timeArray = time.localtime(start) t = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) print('load {} to {} ----------------- day {} {}'.format(start, end, i, t)) i += 1 config = TaskConfig(cluster, self.auth0_id, self.auth0_sec, self.clinic_token, start, end, "./test/data", "{}_metrics_{}_{}.sqlite".format(cluster, start, end), "{}_slowlog_{}_{}.sqlite".format(cluster, start, end), '{}_event_{}_{}.sqlite'.format(cluster, start, end), True, False, {'databaseVersion': 'v6.1.5'}) try: loader = LocalFileLoader('../../test_path') task = DefaultTask(config, loader) task.execute() except: print("get err", sys.exc_info()[1]) # finally: # shutil.rmtree("./test/") class TestReport(unittest.TestCase): @unittest.skipUnless(PLOTLY_AVAILABLE, "plotly package not available") def test_report_generation(self): # nftscan cluster_info1 = { 'cluster_id': '1379661944629574082', 'meta_info': { 'databaseVersion': 'v6.5.2', 'topology': [ { 'name': 'tidb', 'cpu': 8, 'memory': '16GiB', 'storage': '0GiB', 'replicas': 6 }, { 'name': 'tikv', 'cpu': 8, 'memory': '32GiB', 'storage': '1024GiB', 'replicas': 18 }, { 'name': 'tiflash', 'cpu': 8, 'memory': '64GiB', 'storage': '2048GiB', 'replicas': 2 }, { 'name': 'pd', 'cpu': 4, 'memory': '8GiB', 'storage': '20GiB', 'replicas': 3 } ] }, 'tikv_capacity': 18432, 'tiflash_capacity': 4096, 'tiflash_exist': True } # shoplazza cluster_info2 = { 'cluster_id': '1379661944643944089', 'meta_info': { 'databaseVersion': 'v6.1.5', 'topology': [ { 'name': 'tidb', 'cpu': 8, 'memory': '16GiB', 'storage': '0GiB', 'replicas': 4 }, { 'name': 'tikv', 'cpu': 8, 'memory': '32GiB', 'storage': '1024GiB', 'replicas': 6 }, { 'name': 'pd', 'cpu': 4, 'memory': '8GiB', 'storage': '20GiB', 'replicas': 3 } ] }, 'tikv_capacity': 18432, 'tiflash_capacity': 0, 'tiflash_exist': False } # certik cluster_info = { 'cluster_id': '1379661944636804070', 'meta_info': { 'databaseVersion': 'v6.5.0', 'topology': [ { 'name': 'tidb', 'cpu': 8, 'memory': '16GiB', 'storage': '0GiB', 'replicas': 2 }, { 'name': 'tikv', 'cpu': 8, 'memory': '32GiB', 'storage': '4096GiB', 'replicas': 3 }, { 'name': 'pd', 'cpu': 4, 'memory': '8GiB', 'storage': '20GiB', 'replicas': 3 } ] }, 'tikv_capacity': 12288, 'tiflash_capacity': 0, 'tiflash_exist': False } week1_start = 1686585600 # 本周的起始时间零点 week2_start = 1685980800 # 上周的起始时间零点 sqlite_path = './test2/drhouse/data/sqlite' report_path = './test2/drhouse/data/report/{}_v10.html'.format(cluster_info['cluster_id']) report = ReportV2(cluster_info, week1_start, week2_start, sqlite_path, report_path) report.generate_report() class TestNFTScanReport(unittest.TestCase): auth0_id = "-" auth0_sec = "-" clinic_token = '' @unittest.skipUnless(PLOTLY_AVAILABLE, "plotly package not available") def test_nftscan_report(self): os.environ['AWS_PROFILE'] = 'ob-prod-oncall' cluster = os.environ['CLUSTER_ID'] # 1379661944636804070 start_day = int(os.environ['START_DAY_TIME']) # 1687276800 day_length = int(os.environ['DAY_LENGTH']) #14 bucket_name = os.environ['BUCKET_NAME'] # drhouse-slowlog-tmp-shoutheast-1 loader = S3Loader(bucket_name) i = 0 while i < day_length: start = start_day + i*24*60*60 end = start + 24*60*60 timeArray = time.localtime(start) t = time.strftime("%Y-%m-%d %H:%M:%S", timeArray) print('load {} to {} ----------------- day {} {}'.format(start, end, i, t)) i += 1 config = TaskConfig(cluster, self.auth0_id, self.auth0_sec, self.clinic_token, start, end, "./test/data", "{}_metrics_{}_{}.sqlite".format(cluster, start, end), "{}_slowlog_{}_{}.sqlite".format(cluster, start, end), '{}_event_{}_{}.sqlite'.format(cluster, start, end), True, False, {'databaseVersion': 'v6.1.5'}) try: task = DefaultTask(config, loader) task.execute() except: print("get err", sys.exc_info()[1]) cluster_info1 = { 'cluster_id': '1379661944629574082', 'meta_info': { 'databaseVersion': 'v6.5.2', 'topology': [ { 'name': 'tidb', 'cpu': 8, 'memory': '16GiB', 'storage': '0GiB', 'replicas': 6 }, { 'name': 'tikv', 'cpu': 8, 'memory': '32GiB', 'storage': '1024GiB', 'replicas': 18 }, { 'name': 'tiflash', 'cpu': 8, 'memory': '64GiB', 'storage': '2048GiB', 'replicas': 2 }, { 'name': 'pd', 'cpu': 4, 'memory': '8GiB', 'storage': '20GiB', 'replicas': 3 } ] }, 'tikv_capacity': 18432, 'tiflash_capacity': 4096, 'tiflash_exist': True } cluster_info = { 'cluster_id': '1379661944636804070', 'meta_info': { 'databaseVersion': 'v6.5.0', 'topology': [ { 'name': 'tidb', 'cpu': 8, 'memory': '16GiB', 'storage': '0GiB', 'replicas': 2 }, { 'name': 'tikv', 'cpu': 8, 'memory': '32GiB', 'storage': '4096GiB', 'replicas': 3 }, { 'name': 'pd', 'cpu': 4, 'memory': '8GiB', 'storage': '20GiB', 'replicas': 3 } ] }, 'tikv_capacity': 12288, 'tiflash_capacity': 0, 'tiflash_exist': False } week1_start = start_day + 7*24*60*60 # 本周的起始时间零点 week2_start = start_day # 上周的起始时间零点 sqlite_path = './test/data/drhouse/data/sqlite' report = './test/data/drhouse/data/report' if not os.path.exists(report): os.mkdir(report) report_path = './test/data/drhouse/data/report/{}.html'.format(cluster_info['cluster_id']) report = ReportV2(cluster_info, week1_start, week2_start, sqlite_path, report_path) report.generate_report() if os.path.exists(report_path): key = os.path.join(get_report_prefix(), '{}.html'.format(cluster_info['cluster_id'])) loader.upload_file(report_path, key) print("upload report") if __name__ == '__main__': TestNFTScanReport().test() # TestDefaultTask.test_default_task()