# inputs: cluster_id, clinic_org_id, time_range or last_timestamp, apikey # outputs: events(0..N), last_timestamp if not time_range import asyncio import json import os import time from datetime import datetime, timedelta, timezone from uuid import uuid4 import boto3 import pandas as pd import tuotuo from sinks import FullActivityClient from tuotuo.activities import Activity, ActivityType, Sevirity from tuotuo.metrics import QueryParam, QueryRange ACTIVITY_BUCKET = os.environ["ACTIVITY_BUCKET"] # "activities" PATH = "reboot_detector" ACTIVITY_HUB_ENDPOINT = os.environ["ACTIVITY_HUB_ENDPOINT"] SOURCE_KEY = os.environ["SOURCE_KEY"] SOURCE = os.environ["SOURCE"] def dict_to_s3(dict, bucket, key): s3 = boto3.resource("s3") res = s3.Bucket(bucket).put_object(Key=key, Body=json.dumps(dict)) return res async def write_activity_to_hub(activity: Activity): async with FullActivityClient( source=SOURCE, source_key=SOURCE_KEY, endpoint=ACTIVITY_HUB_ENDPOINT ) as client: return await client.write_activity(activity) def cal_falldown(df): if isinstance(df, pd.DataFrame): df = df.interpolate(replicate=True) try: start = next(idx for idx, row in df.iterrows() if (row != 0).all()) end = next(idx for idx, row in df.iloc[::-1].iterrows() if (row != 0).all()) except: return {} temp = df.loc[start - timedelta(0, 0, 0, 0, 0, 15) : end + timedelta(0, 0, 0, 0, 0, 15)] falldows = {} for col_name, item in temp.items(): col_fall = [] for i in range(len(item.tolist()) - 2): if item.tolist()[i + 1] < item.tolist()[i]: col_fall.append( temp.index[i] ) # f"{temp.index[i].replace(tzinfo=timezone.utc).astimezone(tz)}" falldows[col_name] = col_fall return falldows else: return {} def generate_activity(cluster_id, instance, reboot_time, tenant_id, project_id): activity_id = str(uuid4()) occurrence_time = int(reboot_time.timestamp()) detect_time = int(datetime.now().timestamp()) summary = f"{instance} of cluster {cluster_id} was rebooted." details = { "cluster_id": cluster_id, "instance": instance, "tenant_id": tenant_id, "project_id": project_id, "occurrence_time": occurrence_time, "detect_time": detect_time, "evidence": { "source": "Uptime metric", "content": [reboot_time.strftime("%Y-%m-%d %H:%M:%S")], }, } res = dict_to_s3(details, ACTIVITY_BUCKET, f"{PATH}/{activity_id}.json") print(res) payload = { "summary": summary, "cluster_id": cluster_id, "instance": instance, "occurrence_time": occurrence_time, "detect_time": detect_time, "details": { "type": "s3", "bucket": ACTIVITY_BUCKET, "key": f"{PATH}/{activity_id}.json", }, } activity_name = "Reboot" if instance.startswith("db-tidb"): activity_name = "TiDB Reboot" elif instance.startswith("db-tikv"): activity_name = "TiKV Reboot" reboot_event = Activity( activity_type=ActivityType.EVENT, activity_name=activity_name, target_id=cluster_id, calibration_time=occurrence_time, source=SOURCE, source_key=SOURCE_KEY, catalog="Reboot", severity=Sevirity.INFO, activity_id=activity_id, application_id=tenant_id, concept_id=project_id, report_time=detect_time, author="reboot-detector", context_id=activity_id, # context=[], payload=payload, occurrence_time=occurrence_time, detect_time=detect_time, ) return reboot_event async def get_reboot( cluster_id: str, start: datetime, end: datetime, tenant_id: str, project_id: str, clinic_org_id: str, region_name: str, provider: str, ): reboot_events = [] query_range = QueryRange(start, end, "15s") query = 'label_replace(time()-process_start_time_seconds{cluster_id=~".*$tidb_cluster",component=~"ti.*"},"name","$1","instance","(.*)")' query_param = QueryParam( time_arg=query_range, query_template=query, cluster_id=cluster_id, region_name=region_name, clinic_org_id=clinic_org_id, provider=provider, tidb_cluster=cluster_id, query_name="component_uptime:by_instance", query_labels=["instance"], ) client = tuotuo.client( "metrics", ) async with client: result = await client.query(query_param) if result.exception is None: res = cal_falldown(result.frame) for instance, timelist in res.items(): for reboot_time in timelist: reboot_event = generate_activity( cluster_id, instance, reboot_time, tenant_id, project_id ) reboot_events.append(reboot_event) else: print(result.exception) if len(reboot_events) > 0: for reboot_event in reboot_events: res = await write_activity_to_hub(reboot_event) if res.get("status") == "ok": print( f"[Info][Write event Successful] {cluster_id} Reboot activity: {reboot_event}" ) else: print(f"[Error][Write event Failed] {cluster_id} Reboot activity: {reboot_event}") return reboot_events if __name__ == "__main__": # start = 1668341112 # 1666785600 # 2022-10-26 20:00:00+08:00 # end = 1668344112 # 1666799999 start = datetime(2022, 12, 14, 16, 50, 0, tzinfo=timezone(timedelta(hours=8))) end = start + timedelta(0, 0, 0, 1, 30, 0) cluster_id = "1379661944637854363" tenant_id = "1372813089187791289" project_id = "1372813089202551283" proc_start = time.perf_counter() reboot_events = asyncio.run(get_reboot(cluster_id, start, end, tenant_id, project_id)) # for k, v in falldowns.items(): # falldowns[k] = [f"{i.replace(tzinfo=timezone.utc).astimezone(timezone(timedelta(hours=8)))}" for i in v] for reboot_event in reboot_events: print( datetime.fromtimestamp(reboot_event.calibration_time) .replace(tzinfo=timezone.utc) .astimezone(timezone(timedelta(hours=8))), ": ", reboot_event.payload.get("summary"), ) proc_end = time.perf_counter() print(f"Elapsed time: {proc_end - proc_start} seconds")