import os import json import time import urllib.request CE_ENDPOINT = os.environ["CE_ENDPOINT"] API_KEY = os.environ["CLINIC_API_KEY"] CLINIC_ENDPOINT = os.environ["CLINIC_ENDPOINT"] ''' orgs = [ "1311265642040528896", "1326455380288475136", "1372813089189201287", "1372813089191061283", "1372813089193221283", "1372813089189951285", "1372813089192171283", "1372813089188871292", "1372813089193851282", "1372813089187041280", "1372813089195471292", ] ''' # clinic org id changed, which is not equal to dbaas org id orgs = [ "1372813089196911665", "1372813089196911668", "1372813089196912250", "1372813089196915172", "1372813089196915567", "1372813089196912458", "1372813089196915435", "1372813089196912181", "1372813089196915680", "1372813089196911678", "1372813089196916352", ] def log_diag_tasks(org: str, cluster: str, ts: int): step = 60 return [ { 'name': 'log-diag', 'args': { 'cluster': cluster, 'start_ts': ts // step * step - step, 'stop_ts': ts // step * step, 'org_type': 'cloud' }, 'rate': 60, 'offset': 0, } ] def metrics_advisor_tasks(org: str, cluster: str, ts: int): step = 300 # seconds rate = 60 return [ { 'name': 'metrics-advisor', 'args': { 'cluster': cluster, 'start_ts': ts // rate * rate - step, 'stop_ts': ts // rate * rate, 'org_type': 'cloud' }, 'rate': rate, 'offset': 0, } ] def plg_sync_task(ts: int): step = 24 * 60 * 60 rate = 24 * 60 * 60 return [ { 'name': 'plg-data-sync', 'args': { 'time': ts - step }, 'rate': rate, 'offset': 60 * 60 * 1, }, { 'name': 'plg-classify-users', 'args': { 'time': ts, }, 'rate': rate, 'offset': 60 * 60 * 2, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/marketing_report.ipynb', 'target': 'html/marketing_report.html' }, 'rate': rate, 'offset': 60 * 60 * 2, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/sales_report_jp.ipynb', 'target': 'html/sales_report_jp.html' }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/sales_report_na.ipynb', 'target': 'html/sales_report_na.html' }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/sales_report_co.ipynb', 'target': 'html/sales_report_co.html' }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/sales_report_apac.ipynb', 'target': 'html/sales_report_apac.html' }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/sales_report_emea.ipynb', 'target': 'html/sales_report_emea.html' }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/feature_evaluation_report_v1.1.ipynb', 'target': 'html/feature_evaluation_report_v1.1.html' }, 'rate': rate, 'offset': 60 * 60 * 2, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/business_metrics.ipynb', 'target': 'html/business_metrics.html' }, 'rate': rate, 'offset': 60 * 60 * 2, }, { 'name': 'ce-job-html-to-docx', 'args': { "filename": "feature_evaluation_report_v1.1" }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'ce-job-html-to-docx', 'args': { "filename": "business_metrics" }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'ce-job-docx-s3-to-google', 'args': { "report_name": "feature_evaluation_report_v1.1" }, 'rate': rate, 'offset': 60 * 60 * 2.4, }, { 'name': 'ce-job-docx-s3-to-google', 'args': { "report_name": "business_metrics" }, 'rate': rate, 'offset': 60 * 60 * 2.4, }, { 'name': 'pingcap-plg', 'args': { 'bucket': 'pingcap-plg', 'source': 'notebook/sales_report_class1.ipynb', 'target': 'html/sales_report_class1.html' }, 'rate': rate, 'offset': 60 * 60 * 2.3, }, { 'name': 'ce-job-plg-larkbot', 'args': { 'timestamp': ts, }, 'rate': rate, 'offset': 60 * 60 * 2.4, }, ] def task_list(org: str, cluster: str, ts: int): tasks = [] tasks += log_diag_tasks(org, cluster, ts) tasks += metrics_advisor_tasks(org, cluster, ts) return tasks def trigger(task: str, tags: dict, args: dict): try: tags['source'] = 'auto' content = urllib.request.urlopen(urllib.request.Request( f'https://{CE_ENDPOINT}/tasks/{task}/async', method='POST', data=json.dumps({ 'tags': tags, 'args': args, }).encode(), )).read().decode() print(f'trigger task {task} with result: {content}') return content except Exception as e: print(f'trigger task {task} with exception: {e}') return str(e) # remove duplicate tasks def dedup(tasks): xs = [] s = set() for task in tasks: if 'key' not in task: xs.append(task) elif task['key'] not in s: s.add(task['key']) xs.append(task) return xs def lambda_handler(event, context): ts = int(time.time()) tasks = [] # general tasks # tasks += rule_stats_task(ts) # tasks += rule_feedback_records_task(ts) tasks += plg_sync_task(ts) # per-cluster tasks for org in orgs: try: content = urllib.request.urlopen(urllib.request.Request( f'{CLINIC_ENDPOINT}/clinic/api/v1/orgs/{org}/clusters', method='GET', headers={"Authorization": f"Bearer {API_KEY}"}, )).read().decode() content = json.loads(content) for cluster in map(lambda x: x['id'], content['items']): tasks += task_list(org, cluster, ts) except Exception as e: print(f'error on getting tasks for org {org}: {e}') continue executed = [] for task in dedup(tasks): if (ts - task['offset']) % task['rate'] >= 60: continue tags = task['tags'] if 'tags' in task else {} args = task['args'] if 'args' in task else {} r = trigger(task['name'], tags, args) executed.append(dict(**task, result=r)) # To save database space, just print executed tasks for debug instead of store them print('execute', executed) return {}